diff --git a/.github/actions/compilation-cache/action.yml b/.github/actions/compilation-cache/action.yml index 37cff2bba..0b91fd841 100644 --- a/.github/actions/compilation-cache/action.yml +++ b/.github/actions/compilation-cache/action.yml @@ -5,23 +5,31 @@ inputs: description: Compilation owner whose cache should be reused required: false default: ${{ github.job }} + family: + description: Compilation cache family, including fallback isolation + required: false + default: release-ci-v1 + directories: + description: Compilation output directories owned by this job + required: false + default: | + target/debug + target/release + sdk/target/release + sdk/target/sdist/release + target/iceberg-sdk/release + app/crowdb-diskio/build + lib/crowdb-tree/build + lib/crowdb-rpc/build runs: using: composite steps: - uses: Swatinem/rust-cache@v2 with: - shared-key: release-ci-v1-${{ hashFiles('pixi.lock') }} + shared-key: ${{ inputs.family }}-${{ hashFiles('pixi.lock') }} key: ${{ inputs.key }} cache-targets: false - cache-directories: | - target/debug - target/release - sdk/target/release - sdk/target/sdist/release - target/iceberg-sdk/release - app/crowdb-diskio/build - lib/crowdb-tree/build - lib/crowdb-rpc/build + cache-directories: ${{ inputs.directories }} cmd-format: "pixi run {0}" cache-workspace-crates: true env-vars: "CARGO CC CXX CFLAGS CXXFLAGS CMAKE RUST CROWDB_KEEP_DEBUG_INFO" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 180c1fcf4..2eb3cb47d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,8 +63,6 @@ jobs: uses: ./.github/actions/compilation-cache - name: Clean env run: pixi run clean --runtime-only - - name: Build C++ libs - run: pixi run build - name: Run C++ and FFI tests run: pixi run test-cpp - name: Capture runner state on failure @@ -171,8 +169,6 @@ jobs: uses: ./.github/actions/compilation-cache - name: Clean env run: pixi run clean --runtime-only - - name: Build C++ libs (crowdb-diskio subprocess) - run: pixi run build - name: Run storage component tests run: pixi run test-storage - name: Capture runner state on failure @@ -212,6 +208,11 @@ jobs: AccessTests: timeout-minutes: 90 runs-on: ubuntu-24.04 + env: + # Keep function symbols and panic locations without duplicating native + # DWARF across hundreds of linked test executables. + CARGO_PROFILE_TEST_STRIP: debuginfo + CARGO_BUILD_JOBS: "4" steps: - uses: actions/checkout@v4 with: @@ -227,6 +228,12 @@ jobs: with: pixi-version: v0.77.0 - uses: ./.github/actions/compilation-cache + with: + family: access-ci-v2 + directories: | + target/debug + sdk/target/release + sdk/target/sdist/release - name: Clean env run: pixi run clean --runtime-only - name: Run access component tests @@ -658,6 +665,11 @@ jobs: - uses: prefix-dev/setup-pixi@v0.8.1 with: pixi-version: v0.77.0 + - name: Install owned network fault tools + if: matrix.partition == 'console-api' + run: | + pixi run sudo apt-get update + pixi run sudo apt-get install -y iptables - uses: ./.github/actions/restore-compiled with: artifact: ${{ needs.BuildClients.outputs.artifact }} diff --git a/app/crowdb-access-server/src/iceberg/file_http/injection.rs b/app/crowdb-access-server/src/iceberg/file_http/injection.rs index 08f122ee2..432be14ac 100644 --- a/app/crowdb-access-server/src/iceberg/file_http/injection.rs +++ b/app/crowdb-access-server/src/iceberg/file_http/injection.rs @@ -57,6 +57,8 @@ impl FileHttp { ) .await .map_err(|error| { + tracing::warn!(file = %owner.file, %error, + "native injection receive failed; unpublished blocks retained for reclamation"); body.failure() .map_or_else(|| admission_error(error), multipart::encoding_error) })?; @@ -64,11 +66,19 @@ impl FileHttp { .map_err(|_| FileS3ErrorCode::InternalError)? .seal_uploaded(owner, file_request.location.clone(), tree.clone()) .await - .map_err(multipart::seal_error)?; + .map_err(|error| { + tracing::warn!(file = %owner.file, %error, + "native injection seal failed; unpublished blocks retained for reclamation"); + multipart::seal_error(error) + })?; let file = FileRepository::new(self.store.clone()) .finalize_native_parquet(admission.context(), owner, &stem, tree, locations) .await - .map_err(catalog_error)?; + .map_err(|error| { + tracing::warn!(file = %owner.file, %error, + "native injection publication failed; candidate retained for reconciliation"); + catalog_error(error) + })?; let bytes = serde_json::to_vec(&serde_json::json!({"location": file.location.to_string()})) .map_err(|_| FileS3ErrorCode::InternalError)?; let length = bytes.len(); diff --git a/app/crowdb-access-server/src/s3/dependencies.rs b/app/crowdb-access-server/src/s3/dependencies.rs index 201cbf976..57f3a4e23 100644 --- a/app/crowdb-access-server/src/s3/dependencies.rs +++ b/app/crowdb-access-server/src/s3/dependencies.rs @@ -9,6 +9,8 @@ use crowdb_access_s3::{metadata::MetadataKey, metrics::DependencyHealth}; use super::operations::ProductionS3Operations; +pub(super) const PROBE_BUDGET: Duration = Duration::from_secs(2); + pub type DependencyProbeFuture = Pin> + Send + 'static>>; pub struct StorageReadiness { @@ -18,8 +20,8 @@ pub struct StorageReadiness { impl ProductionS3Operations { pub(super) async fn observe_storage(&self) -> StorageReadiness { - let (metadata, chunks) = tokio::join!( - bounded("metadata", async { + observe_storage_dependencies( + async { self.storage .metadata .get( @@ -29,24 +31,50 @@ impl ProductionS3Operations { .await .map_err(|error| error.to_string())?; Ok("Routed Chunk-KV bucket namespace read succeeded".into()) - }), - async { - Box::pin(self.storage.chunks.observe_readiness(Duration::from_millis(750))) + }, + |budget| async move { + Box::pin(self.storage.chunks.observe_readiness(budget)) .await .map_err(|error| error.to_string()) }, - ); - StorageReadiness { metadata, chunks } + ) + .await } + + #[cfg(feature = "test-util")] + /// Exercise the production dependency join and caller budget with controlled reads. + pub async fn observe_storage_dependencies_for_tests(metadata: M, chunks: C) -> StorageReadiness + where + M: Future>, + C: FnOnce(Duration) -> F, + F: Future>, + { + observe_storage_dependencies(metadata, chunks).await + } +} + +async fn observe_storage_dependencies(metadata: M, chunks: C) -> StorageReadiness +where + M: Future>, + C: FnOnce(Duration) -> F, + F: Future>, +{ + let (metadata, chunks) = tokio::join!(bounded("metadata", metadata), chunks(PROBE_BUDGET)); + StorageReadiness { metadata, chunks } } pub(super) async fn bounded( dependency: &str, probe: impl Future>, ) -> Result { - tokio::time::timeout(Duration::from_millis(750), probe) + tokio::time::timeout(PROBE_BUDGET, probe) .await - .unwrap_or_else(|_| Err(format!("{dependency} read exceeded its 750 ms deadline"))) + .unwrap_or_else(|_| { + Err(format!( + "{dependency} read exceeded its {} ms deadline", + PROBE_BUDGET.as_millis() + )) + }) } pub(super) fn health(result: &Result) -> DependencyHealth { diff --git a/app/crowdb-access-server/src/s3/health.rs b/app/crowdb-access-server/src/s3/health.rs index 549a28767..33269187b 100644 --- a/app/crowdb-access-server/src/s3/health.rs +++ b/app/crowdb-access-server/src/s3/health.rs @@ -41,6 +41,16 @@ impl S3HttpHandler for AccessHealthHandler { } } +impl S3Dispatcher { + #[cfg(feature = "test-util")] + /// Read the production cached response without transport or dependency work. + #[must_use] + pub fn health_response_for_tests(&self) -> Response { + let observed = self.observations.latest.load_full(); + readiness_response(observed.as_deref(), self.health.is_live(), false) + } +} + fn readiness_response( observed: Option<&HealthObservation>, live: bool, diff --git a/app/crowdb-access-server/src/s3/health/observation.rs b/app/crowdb-access-server/src/s3/health/observation.rs index 61fb9b834..f1b2fb499 100644 --- a/app/crowdb-access-server/src/s3/health/observation.rs +++ b/app/crowdb-access-server/src/s3/health/observation.rs @@ -17,6 +17,8 @@ use tokio::time::{Instant, MissedTickBehavior}; use super::super::{dependencies, S3Dispatcher, StorageReadiness}; +const OBSERVATION_BUDGET: Duration = dependencies::PROBE_BUDGET.saturating_add(Duration::from_millis(100)); + pub(super) const MAX_AGE: Duration = Duration::from_secs(3); #[derive(Default)] @@ -82,16 +84,43 @@ impl S3Dispatcher { u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) }); let observed = tokio::time::timeout( - Duration::from_millis(800), + OBSERVATION_BUDGET, Arc::clone(&self.operations).observe_dependencies(), ) .await .unwrap_or_else(|_| { Some(StorageReadiness { - metadata: Err("Dependency observation exceeded its 800 ms deadline".into()), - chunks: Err("Dependency observation exceeded its 800 ms deadline".into()), + metadata: Err(format!( + "Dependency observation exceeded its {} ms deadline", + OBSERVATION_BUDGET.as_millis() + )), + chunks: Err(format!( + "Dependency observation exceeded its {} ms deadline", + OBSERVATION_BUDGET.as_millis() + )), }) }); + let generation = self + .observations + .latest + .load() + .as_ref() + .map_or(1, |last| last.generation + 1); + if let Some(failed) = observed + .as_ref() + .filter(|value| value.metadata.is_err() || value.chunks.is_err()) + { + tracing::warn!( + generation, + configured_budget_ms = u64::try_from(dependencies::PROBE_BUDGET.as_millis()).unwrap_or(u64::MAX), + observation_budget_ms = u64::try_from(OBSERVATION_BUDGET.as_millis()).unwrap_or(u64::MAX), + total_elapsed_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX), + checked_at_unix_ms, + metadata_error = ?failed.metadata.as_ref().err(), + chunks_error = ?failed.chunks.as_ref().err(), + "Access dependency observation failed; background sampling continues" + ); + } let mut health = self .health .snapshot(&self.metrics, self.body_allocator.as_deref()); @@ -113,12 +142,6 @@ impl S3Dispatcher { DependencyHealth::Unavailable }; } - let generation = self - .observations - .latest - .load() - .as_ref() - .map_or(1, |last| last.generation + 1); HealthObservation { health, details, diff --git a/app/crowdb-access-server/tests/access_probe_budget_test.rs b/app/crowdb-access-server/tests/access_probe_budget_test.rs new file mode 100644 index 000000000..68694babf --- /dev/null +++ b/app/crowdb-access-server/tests/access_probe_budget_test.rs @@ -0,0 +1,136 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. +#![cfg(feature = "s3")] + +#[path = "common/access_probe_budget.rs"] +mod fixture; +use fixture::{advance, failure_warnings, settle, TestSampler, HELD_OUTER}; +use std::sync::atomic::Ordering; + +#[tokio::test(start_paused = true)] +async fn production_metadata_and_chunk_probes_accept_reads_over_750ms_within_two_seconds() { + let (warnings, _subscriber) = failure_warnings(); + let fixture = TestSampler::new(1200); + settle().await; + assert_eq!(fixture.operations.calls.load(Ordering::SeqCst), 1); + advance(750).await; + assert_eq!( + fixture.generation(), + 0, + "a valid slower dependency must remain in progress" + ); + for _ in 0..32 { + assert_eq!(fixture.read(503).await["observation"]["state"], "pending"); + } + assert_eq!(fixture.operations.calls.load(Ordering::SeqCst), 1); + advance(450).await; + let ready = fixture.read(200).await; + assert_eq!(ready["metadata"], "ready"); + assert_eq!(ready["chunks"], "ready"); + assert_eq!(ready["observation"]["age_ms"], 1200); + assert_eq!(fixture.operations.metrics.snapshot().chunkdb.errors, 0); + assert_eq!(fixture.operations.metrics.snapshot().diskio.errors, 0); + assert_eq!(warnings.count.load(Ordering::SeqCst), 0); +} + +#[tokio::test(start_paused = true)] +async fn two_second_failure_is_visible_then_the_same_sampler_recovers_automatically() { + let (warnings, _subscriber) = failure_warnings(); + let fixture = TestSampler::new(3000); + settle().await; + advance(1999).await; + assert_eq!(fixture.generation(), 0); + fixture.operations.delay_ms.store(100, Ordering::SeqCst); + advance(1).await; + let failed = fixture.read(503).await; + assert_eq!(failed["observation"]["generation"], 1); + assert_eq!(failed["metadata"], "unavailable"); + assert_eq!(failed["chunks"], "unavailable"); + assert!(failed["details"]["metadata"] + .as_str() + .unwrap() + .contains("2000 ms")); + assert!(failed["details"]["chunks"] + .as_str() + .unwrap() + .contains("ChunkDB read exceeded")); + assert_eq!(fixture.operations.metrics.snapshot().chunkdb.errors, 1); + assert_eq!(warnings.count.load(Ordering::SeqCst), 1); + assert_eq!(warnings.generation.load(Ordering::SeqCst), 1); + assert_eq!(warnings.budget_ms.load(Ordering::SeqCst), 2000); + assert_eq!(warnings.elapsed_ms.load(Ordering::SeqCst), 2000); + assert!(warnings.checked_at_unix_ms.load(Ordering::SeqCst) > 0); + assert!(fixture.dispatcher.start_health_observer().is_err()); + advance(100).await; + let ready = fixture.read(200).await; + assert_eq!(ready["observation"]["generation"], 2); + assert_eq!(ready["ready"], true); + assert_eq!(fixture.operations.calls.load(Ordering::SeqCst), 2); + assert_eq!(fixture.operations.peak.load(Ordering::SeqCst), 1); + assert_eq!(fixture.operations.cancelled.load(Ordering::SeqCst), 0); + assert_eq!(warnings.count.load(Ordering::SeqCst), 1); +} + +#[tokio::test(start_paused = true)] +async fn outer_watchdog_cancels_at_2100ms_then_sampling_continues_without_overlap_or_burst() { + let (warnings, _subscriber) = failure_warnings(); + let mut fixture = TestSampler::new(HELD_OUTER); + settle().await; + advance(2099).await; + assert_eq!(fixture.generation(), 0); + fixture.operations.delay_ms.store(100, Ordering::SeqCst); + advance(1).await; + let failed = fixture.read(503).await; + assert!(failed["details"]["metadata"] + .as_str() + .unwrap() + .contains("2100 ms")); + assert_eq!(fixture.operations.cancelled.load(Ordering::SeqCst), 1); + assert_eq!(warnings.count.load(Ordering::SeqCst), 1); + assert_eq!(warnings.elapsed_ms.load(Ordering::SeqCst), 2100); + assert_eq!(fixture.operations.calls.load(Ordering::SeqCst), 2); + assert_eq!(fixture.operations.active.load(Ordering::SeqCst), 1); + advance(100).await; + assert_eq!(fixture.read(200).await["observation"]["generation"], 2); + advance(799).await; + assert_eq!(fixture.operations.calls.load(Ordering::SeqCst), 2); + advance(1).await; + assert_eq!(fixture.operations.calls.load(Ordering::SeqCst), 3); + assert_eq!(fixture.operations.peak.load(Ordering::SeqCst), 1); + fixture.observer.take(); + settle().await; + assert_eq!(fixture.operations.active.load(Ordering::SeqCst), 0); + assert_eq!(fixture.operations.disk.active.load(Ordering::SeqCst), 0); + assert_eq!(fixture.operations.metrics.snapshot().chunkdb.inflight, 0); + assert_eq!(fixture.operations.metrics.snapshot().diskio.inflight, 0); + assert_eq!(warnings.count.load(Ordering::SeqCst), 1); +} + +#[tokio::test(start_paused = true)] +async fn consecutive_slow_successes_keep_the_three_second_start_age_expiry_and_recover() { + let (warnings, _subscriber) = failure_warnings(); + let fixture = TestSampler::new(1800); + settle().await; + advance(1800).await; + let first = fixture.read(200).await; + assert_eq!(first["observation"]["generation"], 1); + assert_eq!(first["observation"]["max_age_ms"], 3000); + advance(1200).await; + assert_eq!(fixture.read(200).await["observation"]["age_ms"], 3000); + advance(1).await; + let stale = fixture.read(503).await; + assert_eq!(stale["observation"]["state"], "expired"); + assert_eq!(stale["metadata"], "unknown"); + assert_eq!(stale["chunks"], "unknown"); + assert_eq!( + stale["observation"]["checked_at_unix_ms"], + first["observation"]["checked_at_unix_ms"] + ); + assert_eq!(stale["observation"]["generation"], 1); + advance(599).await; + let recovered = fixture.read(200).await; + assert_eq!(recovered["observation"]["generation"], 2); + assert_eq!(recovered["observation"]["age_ms"], 1800); + assert_eq!(fixture.operations.peak.load(Ordering::SeqCst), 1); + assert_eq!(warnings.count.load(Ordering::SeqCst), 0); +} diff --git a/app/crowdb-access-server/tests/common/access_probe_budget.rs b/app/crowdb-access-server/tests/common/access_probe_budget.rs new file mode 100644 index 000000000..5bacf305b --- /dev/null +++ b/app/crowdb-access-server/tests/common/access_probe_budget.rs @@ -0,0 +1,255 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use async_trait::async_trait; +use bytes::Bytes; +use crowdb_access_s3::{ + auth::TrustedNetworkAuthenticator, + metrics::{DependencyHealth, S3Health, S3Metrics}, + route::S3Route, +}; +use crowdb_access_server::s3::{ + DependencyProbeFuture, HealthObserver, ProductionS3Operations, S3Dispatcher, S3Operations, + S3OperationsFuture, +}; +use crowdb_chunk_client::{metrics::ReadinessMetrics, ChunkIoClient, DiskWriter, Result}; +use crowdb_kv_client::DiskRecord; +use crowdb_protocol::{ + common::{DiskId, HwStatus}, + diskdb::rpc::{DiskValue, Segment}, +}; +use http_body_util::BodyExt; +use hyper::{body::Incoming, Request}; +use serde_json::Value; +use std::{ + sync::{ + atomic::{AtomicU64, AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::time::Instant; + +pub const HELD_OUTER: u64 = u64::MAX; + +#[derive(Default)] +pub struct TestDisk { + pub active: AtomicUsize, +} + +struct TestRead<'a>(&'a AtomicUsize); +impl Drop for TestRead<'_> { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::SeqCst); + } +} + +#[async_trait] +impl DiskWriter for TestDisk { + async fn write(&self, _: &Segment, _: u64, _: Bytes) -> Result<()> { + panic!("probe must not write"); + } + async fn write_at_byte_offset(&self, _: &Segment, _: u64, _: u64, _: Bytes) -> Result<()> { + panic!("probe must not write"); + } + async fn read_priority(&self, segment: &Segment, unit: u64, offset: u64, length: u32) -> Result { + assert!(segment.disk_id.is_some()); + assert_eq!((unit, offset, length), (4096, 0, 4096)); + self.active.fetch_add(1, Ordering::SeqCst); + let _read = TestRead(&self.active); + tokio::time::sleep(Duration::from_millis(100)).await; + Ok(Bytes::from(vec![7; 4096])) + } +} + +pub struct TestOperations { + pub delay_ms: AtomicU64, + pub calls: AtomicUsize, + pub active: AtomicUsize, + pub peak: AtomicUsize, + pub cancelled: AtomicUsize, + pub metrics: Arc, + pub disk: Arc, +} + +struct TestRound<'a> { + state: &'a TestOperations, + completed: bool, +} +impl Drop for TestRound<'_> { + fn drop(&mut self) { + self.state.active.fetch_sub(1, Ordering::SeqCst); + if !self.completed { + self.state.cancelled.fetch_add(1, Ordering::SeqCst); + } + } +} + +fn disks() -> Vec { + vec![DiskRecord { + rack_id: 1, + node_id: 1, + disk_group_id: 1, + disk_id: DiskId { high: 1, low: 1 }, + value: DiskValue { + status: HwStatus::Up as i32, + ..Default::default() + }, + }] +} + +impl S3Operations for TestOperations { + fn observe_dependencies(self: Arc) -> DependencyProbeFuture { + Box::pin(async move { + self.calls.fetch_add(1, Ordering::SeqCst); + let count = self.active.fetch_add(1, Ordering::SeqCst) + 1; + self.peak.fetch_max(count, Ordering::SeqCst); + let mut round = TestRound { + state: self.as_ref(), + completed: false, + }; + let delay = self.delay_ms.load(Ordering::SeqCst); + if delay == HELD_OUTER { + std::future::pending::<()>().await; + } + let metrics = self.metrics.clone(); + let disk = self.disk.clone(); + let observed = ProductionS3Operations::observe_storage_dependencies_for_tests( + async move { + tokio::time::sleep(Duration::from_millis(delay)).await; + Ok("current metadata read".into()) + }, + move |budget| async move { + let online = ChunkIoClient::observe_readiness_dependencies_for_tests( + Instant::now() + budget, + &metrics, + disk, + async { Ok(disks()) }, + async move { + tokio::time::sleep(Duration::from_millis(delay)).await; + Ok(()) + }, + ) + .await + .map_err(|error| error.to_string())?; + Ok(format!("current physical read on {online} disk")) + }, + ) + .await; + round.completed = true; + Some(observed) + }) + } + fn execute( + self: Arc, + _: S3Route, + _: Request, + _: String, + _: String, + ) -> S3OperationsFuture { + panic!("health must not execute application requests"); + } +} + +pub struct TestSampler { + pub dispatcher: Arc, + pub operations: Arc, + pub observer: Option, +} +impl TestSampler { + pub fn new(delay_ms: u64) -> Self { + let operations = Arc::new(TestOperations { + delay_ms: AtomicU64::new(delay_ms), + calls: AtomicUsize::new(0), + active: AtomicUsize::new(0), + peak: AtomicUsize::new(0), + cancelled: AtomicUsize::new(0), + metrics: Arc::default(), + disk: Arc::default(), + }); + let health = Arc::new(S3Health::starting(1)); + health.set_listener(DependencyHealth::Ready); + let dispatcher = Arc::new( + S3Dispatcher::new( + Arc::new(TrustedNetworkAuthenticator::new()), + operations.clone(), + Arc::new(S3Metrics::default()), + "probe-test".into(), + false, + ) + .with_health(health), + ); + let observer = Some(dispatcher.start_health_observer().unwrap()); + Self { + dispatcher, + operations, + observer, + } + } + pub async fn read(&self, status: u16) -> Value { + let response = self.dispatcher.health_response_for_tests(); + assert_eq!(response.status().as_u16(), status); + serde_json::from_slice(&response.into_body().collect().await.unwrap().to_bytes()).unwrap() + } + pub fn generation(&self) -> u64 { + self.dispatcher.health_generation_for_tests() + } +} +impl Drop for TestSampler { + fn drop(&mut self) { + self.observer.take(); + } +} + +pub async fn settle() { + for _ in 0..8 { + tokio::task::yield_now().await; + } +} +pub async fn advance(milliseconds: u64) { + tokio::time::advance(Duration::from_millis(milliseconds)).await; + settle().await; +} + +#[derive(Default)] +pub struct TestFailureWarnings { + pub count: AtomicU64, + pub generation: AtomicU64, + pub budget_ms: AtomicU64, + pub elapsed_ms: AtomicU64, + pub checked_at_unix_ms: AtomicU64, +} + +struct TestWarningLayer(Arc); +struct TestWarningFields<'a>(&'a TestFailureWarnings); +impl tracing::field::Visit for TestWarningFields<'_> { + fn record_debug(&mut self, _: &tracing::field::Field, _: &dyn std::fmt::Debug) {} + fn record_u64(&mut self, field: &tracing::field::Field, value: u64) { + let target = match field.name() { + "generation" => &self.0.generation, + "configured_budget_ms" => &self.0.budget_ms, + "total_elapsed_ms" => &self.0.elapsed_ms, + "checked_at_unix_ms" => &self.0.checked_at_unix_ms, + _ => return, + }; + target.store(value, Ordering::SeqCst); + } +} +impl tracing_subscriber::Layer for TestWarningLayer { + fn on_event(&self, event: &tracing::Event<'_>, _: tracing_subscriber::layer::Context<'_, S>) { + if event.metadata().level() == &tracing::Level::WARN + && event.metadata().fields().field("configured_budget_ms").is_some() + { + self.0.count.fetch_add(1, Ordering::SeqCst); + event.record(&mut TestWarningFields(self.0.as_ref())); + } + } +} + +pub fn failure_warnings() -> (Arc, tracing::subscriber::DefaultGuard) { + use tracing_subscriber::prelude::*; + let observed = Arc::new(TestFailureWarnings::default()); + let subscriber = tracing_subscriber::registry().with(TestWarningLayer(observed.clone())); + let guard = tracing::subscriber::set_default(subscriber); + (observed, guard) +} diff --git a/app/crowdb-access-server/tests/common/iceberg_client.py b/app/crowdb-access-server/tests/common/iceberg_client.py index 684f5eb5d..0d1a7abe7 100644 --- a/app/crowdb-access-server/tests/common/iceberg_client.py +++ b/app/crowdb-access-server/tests/common/iceberg_client.py @@ -60,7 +60,7 @@ def main(): def verify_namespaces(uri, properties): writer = load_catalog("crowdb", **(properties | {"token": "w" * 32})) namespace = ("client-" + uuid.uuid4().hex,) - child = namespace + ("冰+a%2F",) + child = namespace + ("\u51b0+a%2F",) writer.create_namespace(namespace, {"owner": "original"}) assert writer.namespace_exists(namespace) assert writer.load_namespace_properties(namespace) == {"owner": "original"} diff --git a/app/crowdb-access-server/tests/dataset_batch_header_test.rs b/app/crowdb-access-server/tests/dataset_batch_header_test.rs index 603fdac1e..cb633204b 100644 --- a/app/crowdb-access-server/tests/dataset_batch_header_test.rs +++ b/app/crowdb-access-server/tests/dataset_batch_header_test.rs @@ -12,23 +12,24 @@ const SNAPSHOT: &str = "11111111111111111111111111111111"; #[test] fn ordered_cells_preserve_unicode_missing_tombstone_empty_and_chunk_wire_bytes() { - let (prefix, metadata) = DatasetHttpService::batch_header_for_tests(resolved(Some("游标\"/next"))) - .expect("bounded batch header"); + let (prefix, metadata) = + DatasetHttpService::batch_header_for_tests(resolved(Some("\u{6e38}\u{6807}\"/next"))) + .expect("bounded batch header"); let expected = concat!( "{\"snapshot\":\"11111111111111111111111111111111\",", "\"sample_ids\":[\"YQ==\",\"+/8=\"],\"fields\":[", - "{\"sample\":\"YQ==\",\"field\":\"空\",\"state\":\"value\",\"length\":0,", + "{\"sample\":\"YQ==\",\"field\":\"\u{7a7a}\",\"state\":\"value\",\"length\":0,", "\"md5\":\"d41d8cd98f00b204e9800998ecf8427e\"},", "{\"sample\":\"YQ==\",\"field\":\"erased\",\"state\":\"tombstone\",\"length\":0,\"md5\":null},", "{\"sample\":\"YQ==\",\"field\":\"absent\",\"state\":\"missing\",\"length\":0,\"md5\":null},", "{\"sample\":\"YQ==\",\"field\":\"blob\",\"state\":\"value\",\"length\":1,", "\"md5\":\"9dd4e461268c8034f5c8564e155c67a6\"},", - "{\"sample\":\"+/8=\",\"field\":\"空\",\"state\":\"missing\",\"length\":0,\"md5\":null},", + "{\"sample\":\"+/8=\",\"field\":\"\u{7a7a}\",\"state\":\"missing\",\"length\":0,\"md5\":null},", "{\"sample\":\"+/8=\",\"field\":\"erased\",\"state\":\"missing\",\"length\":0,\"md5\":null},", "{\"sample\":\"+/8=\",\"field\":\"absent\",\"state\":\"missing\",\"length\":0,\"md5\":null},", "{\"sample\":\"+/8=\",\"field\":\"blob\",\"state\":\"value\",\"length\":4294967297,", "\"md5\":\"abababababababababababababababab\"}],", - "\"next\":\"游标\\\"/next\",\"end\":false}" + "\"next\":\"\u{6e38}\u{6807}\\\"/next\",\"end\":false}" ); assert_wire(&prefix, &metadata, expected); } @@ -39,7 +40,7 @@ fn terminal_header_keeps_sample_and_field_order_without_a_cursor() { assert!(metadata.ends_with(b",\"next\":null,\"end\":true}")); let value: serde_json::Value = serde_json::from_slice(&metadata).unwrap(); assert_eq!(value["sample_ids"], serde_json::json!(["YQ==", "+/8="])); - assert_eq!(value["fields"][0]["field"], "空"); + assert_eq!(value["fields"][0]["field"], "\u{7a7a}"); assert_eq!(value["fields"][7]["sample"], "+/8="); assert_eq!(value["next"], serde_json::Value::Null); assert_eq!(value["end"], true); @@ -88,12 +89,14 @@ fn assert_wire(prefix: &Bytes, metadata: &Bytes, expected: &str) { fn resolved(next: Option<&str>) -> ResolvedReadPage { let mut value = empty(next.map(str::to_owned)); value.page.ids = vec![b"a".to_vec(), vec![0xfb, 0xff]]; - value.page.fields = ["空", "erased", "absent", "blob"].map(str::to_owned).to_vec(); + value.page.fields = ["\u{7a7a}", "erased", "absent", "blob"] + .map(str::to_owned) + .to_vec(); value.samples = vec![ SampleView { sample_id: b"a".to_vec(), fields: BTreeMap::from([ - ("空".into(), inline(b"")), + ("\u{7a7a}".into(), inline(b"")), ("erased".into(), FieldLocator::Tombstone), ("blob".into(), inline(b"x")), ]), diff --git a/app/crowdb-access-server/tests/iceberg_file_request_test.rs b/app/crowdb-access-server/tests/iceberg_file_request_test.rs index 739d7940d..7e073253b 100644 --- a/app/crowdb-access-server/tests/iceberg_file_request_test.rs +++ b/app/crowdb-access-server/tests/iceberg_file_request_test.rs @@ -117,7 +117,7 @@ fn native_list_requests_preserve_table_prefix_and_validate_all_selectors() { .unwrap() .unwrap(); assert_eq!(list.table, table); - assert_eq!(list.prefix, format!("{}data/雪%2B", table.object_prefix())); + assert_eq!(list.prefix, format!("{}data/\u{96ea}%2B", table.object_prefix())); assert_eq!(list.delimiter.as_deref(), Some("/")); assert!(list.encoding_url); assert_eq!(list.max_keys, 3); @@ -173,7 +173,7 @@ fn native_object_routes_decode_once_and_preserve_plus_percent_and_repeated_slash ("a+b", "a+b"), ("a%252Fb", "a%2Fb"), ("a//b", "a//b"), - ("%E5%86%B0/a%20b", "冰/a b"), + ("%E5%86%B0/a%20b", "\u{51b0}/a b"), ] { for (method, operation) in [ (Method::GET, FileOperation::Get), diff --git a/app/crowdb-access-server/tests/iceberg_namespace_limits_test.rs b/app/crowdb-access-server/tests/iceberg_namespace_limits_test.rs index a7d065a9a..a35be7263 100644 --- a/app/crowdb-access-server/tests/iceberg_namespace_limits_test.rs +++ b/app/crowdb-access-server/tests/iceberg_namespace_limits_test.rs @@ -79,8 +79,8 @@ async fn property_cardinality_replacement_and_overlap_are_atomic_over_http() { #[tokio::test] async fn property_utf8_key_and_value_byte_limits_fail_without_mutation() { let test = TestTableHttp::new().await; - let key = format!("{}a", "键".repeat(341)); - let value = "值".repeat(2730) + "ab"; + let key = format!("{}a", "\u{952e}".repeat(341)); + let value = "\u{503c}".repeat(2730) + "ab"; update(&test, json!({"updates":{key.clone():value.clone()}}), 200).await; assert_eq!(properties(&test).await, json!({key.clone():value.clone()})); for (invalid_key, invalid_value) in [ diff --git a/app/crowdb-access-server/tests/s3_object_inspection_test.rs b/app/crowdb-access-server/tests/s3_object_inspection_test.rs index bfde1d356..dea267ab2 100644 --- a/app/crowdb-access-server/tests/s3_object_inspection_test.rs +++ b/app/crowdb-access-server/tests/s3_object_inspection_test.rs @@ -163,7 +163,7 @@ async fn inspection_fixture() -> (Arc, S3Dispatcher) { .collect(); let object = ObjectRecord { bucket_id, - key: "folder/中文.bin".as_bytes().to_vec(), + key: "folder/\u{4e2d}\u{6587}.bin".as_bytes().to_vec(), logical_length: 184, checksum: vec![1; 16], etag: "etag".into(), @@ -222,7 +222,7 @@ async fn privileged_http_inspection_reads_two_metadata_keys_and_rejects_stale_ge ); let server = tokio::spawn(serve(listener, Arc::new(dispatcher), std::future::pending())); let http = reqwest::Client::new(); - let query = [("bucket", "bucket"), ("key", "folder/中文.bin")]; + let query = [("bucket", "bucket"), ("key", "folder/\u{4e2d}\u{6587}.bin")]; let response = http.get(&endpoint).query(&query).send().await.unwrap(); assert_eq!(response.status().as_u16(), 401); let response = http diff --git a/app/crowdb-chunk-kv-server/src/catalog/recovery.rs b/app/crowdb-chunk-kv-server/src/catalog/recovery.rs index 85a8feb60..f8ecbd0cf 100644 --- a/app/crowdb-chunk-kv-server/src/catalog/recovery.rs +++ b/app/crowdb-chunk-kv-server/src/catalog/recovery.rs @@ -8,6 +8,24 @@ use crowdb_protocol::chunk_kv::{ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage use std::{fmt::Display, future::Future, time::Instant}; impl ChunkKvRangeCatalogPublisher { + /// Recovers this instance's assignments against the current published head. + /// A successful replay remains usable when only remote assignments change; + /// every local entry must still match exactly, including its epoch and artifact. + /// # Errors + /// Returns catalog or artifact recovery failures under the supplied deadline. + pub async fn recover_current_for_instance( + &self, + instance_id: u64, + deadline: Instant, + recover: F, + ) -> Result, T)>, ChunkKvRangeCatalogError> + where + F: FnMut(Vec) -> R, + R: Future>, + { + self.recover_matching(deadline, Some(instance_id), recover).await + } + /// Recover a published catalog, discarding obsolete recovery results only /// when a newer authoritative generation proves publication advanced. /// @@ -17,6 +35,19 @@ impl ChunkKvRangeCatalogPublisher { pub async fn recover_current( &self, deadline: Instant, + recover: F, + ) -> Result, T)>, ChunkKvRangeCatalogError> + where + F: FnMut(Vec) -> R, + R: Future>, + { + self.recover_matching(deadline, None, recover).await + } + + async fn recover_matching( + &self, + deadline: Instant, + instance_id: Option, mut recover: F, ) -> Result, T)>, ChunkKvRangeCatalogError> where @@ -31,6 +62,17 @@ impl ChunkKvRangeCatalogPublisher { .as_ref() .is_some_and(|(next, _)| next.generation > head.generation) { + if result.is_ok() + && instance_id.is_some_and(|instance| { + let (_, next_pages) = latest.as_ref().expect("newer catalog exists"); + local_assignments_match(&pages, next_pages, instance) + }) + { + let (head, pages) = latest.expect("newer catalog exists"); + return result + .map(|recovered| Some((head, pages, recovered))) + .map_err(|error| ChunkKvRangeCatalogError::Unavailable(error.to_string())); + } if Instant::now() >= deadline { return Err(ChunkKvRangeCatalogError::GenerationConflict); } @@ -47,3 +89,18 @@ impl ChunkKvRangeCatalogPublisher { Ok(None) } } + +fn local_assignments_match( + before: &[ChunkKvRangeCatalogPage], + after: &[ChunkKvRangeCatalogPage], + instance_id: u64, +) -> bool { + let local = |entry: &&crowdb_protocol::chunk_kv::ChunkKvRangeCatalogEntry| { + crate::server::recoverable_local_entry(entry, instance_id) + }; + before + .iter() + .flat_map(|page| &page.entries) + .filter(local) + .eq(after.iter().flat_map(|page| &page.entries).filter(local)) +} diff --git a/app/crowdb-chunk-kv-server/src/main.rs b/app/crowdb-chunk-kv-server/src/main.rs index bd8edb6d2..8320ab374 100644 --- a/app/crowdb-chunk-kv-server/src/main.rs +++ b/app/crowdb-chunk-kv-server/src/main.rs @@ -219,7 +219,8 @@ async fn main() -> std::process::ExitCode { let recovery_service = &service; let instance_id = config.instance_id; match catalog - .recover_current( + .recover_current_for_instance( + instance_id, std::time::Instant::now() + std::time::Duration::from_secs(60), |pages| async move { recover_assigned_partitions(recovery_storage, recovery_service, &pages, instance_id).await @@ -589,10 +590,12 @@ async fn main() -> std::process::ExitCode { refresh_task, )); info!("chunk KV background workers stopped"); - if let Err(error) = service.drain_writers().await { - warn!(%error, "WAL writer drain incomplete; restart must retain lease exclusion"); + match service.drain_writers().await { + Ok(()) => info!("chunk KV WAL writer drain complete"), + Err(error) => { + warn!(%error, "WAL writer drain incomplete; restart must retain lease exclusion"); + } } - info!("chunk KV WAL writer drain complete"); if let Err(error) = process_lease.withdraw_observation().await { warn!(%error, "failed to unregister chunk KV instance"); } diff --git a/app/crowdb-chunk-kv-server/src/server.rs b/app/crowdb-chunk-kv-server/src/server.rs index 957c8c332..60480a628 100644 --- a/app/crowdb-chunk-kv-server/src/server.rs +++ b/app/crowdb-chunk-kv-server/src/server.rs @@ -299,13 +299,6 @@ impl ChunkKvService { self.authority.monotonic_ms() } - /// Stops new admission and revokes successful serving replies. Admitted - /// operations retain their partition handle and can finish durable work. - pub fn begin_drain(&self) { - self.admitting.store(false, Ordering::Release); - self.authority.clear(); - } - #[must_use] pub fn health(&self, now_monotonic_ms: u64) -> ServerHealth { let catalog = self.catalog.load(); @@ -1391,7 +1384,7 @@ fn scan_page_bytes(page: &crowdb_chunk_kv::ScanPage) -> usize { }) } -fn recoverable_local_entry(entry: &ChunkKvRangeCatalogEntry, instance_id: u64) -> bool { +pub(crate) fn recoverable_local_entry(entry: &ChunkKvRangeCatalogEntry, instance_id: u64) -> bool { entry.owner.instance_id == instance_id && !matches!( entry.state, diff --git a/app/crowdb-chunk-kv-server/src/server/drain.rs b/app/crowdb-chunk-kv-server/src/server/drain.rs index c489a6d3f..ea1b7b427 100644 --- a/app/crowdb-chunk-kv-server/src/server/drain.rs +++ b/app/crowdb-chunk-kv-server/src/server/drain.rs @@ -1,15 +1,26 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +use std::sync::atomic::Ordering; + use super::{ChunkKvError, ChunkKvService}; impl ChunkKvService { + /// Stops new admission and revokes successful serving replies. Admitted + /// operations retain their partition handle and can finish durable work. + pub fn begin_drain(&self) { + self.admitting.store(false, Ordering::Release); + self.load_sampling.close(); + self.authority.clear(); + } + /// Drain admitted mutations and release each hosted WAL writer for a new boot. /// Background assignment changes must be stopped before calling this method. /// # Errors /// Leaves uncertain or fenced writers subject to their normal lease exclusion. pub async fn drain_writers(&self) -> Result<(), ChunkKvError> { self.begin_drain(); + self.load_sampling.drain().await; let partitions = self.partitions.load_full(); let mut first_error = None; for partition in partitions.values() { diff --git a/app/crowdb-chunk-kv-server/src/server/load_sampling.rs b/app/crowdb-chunk-kv-server/src/server/load_sampling.rs index 04fc2e84a..90ec1d9a6 100644 --- a/app/crowdb-chunk-kv-server/src/server/load_sampling.rs +++ b/app/crowdb-chunk-kv-server/src/server/load_sampling.rs @@ -5,32 +5,82 @@ use std::collections::HashMap; use std::sync::{ - atomic::{AtomicBool, Ordering}, + atomic::{AtomicU8, Ordering}, Arc, }; use arc_swap::ArcSwap; use crowdb_chunk_kv::{ChunkKvError, Partition, PartitionLifecycle}; use crowdb_protocol::chunk_kv::Id128; +use tokio::sync::Notify; use tracing::warn; type Samples = Vec<(Vec, u64)>; +const RUNNING: u8 = 1; +const CLOSED: u8 = 2; + #[derive(Default)] pub(super) struct LoadSampling { - running: AtomicBool, + state: AtomicU8, cached: ArcSwap>, + completed: Notify, } struct SamplingJob(Arc); impl Drop for SamplingJob { fn drop(&mut self) { - self.0.running.store(false, Ordering::Release); + self.0.state.fetch_and(!RUNNING, Ordering::AcqRel); + self.0.completed.notify_waiters(); + } +} + +// Field order also keeps completion last if queued work is dropped before entry. +struct SamplingWork { + partitions: Arc>, + runtime: tokio::runtime::Handle, + completion: SamplingJob, +} + +impl SamplingWork { + fn run(self, limit: usize) { + // Declared first and completed last, including during unwinding. + let completion = self.completion; + let runtime_handle = self.runtime; + let owned_partitions = self.partitions; + runtime_handle.block_on(async { + let mut observations = HashMap::new(); + for (id, partition) in owned_partitions.iter() { + if completion.0.is_closed() { + return; + } + let snapshot = partition.snapshot(); + if snapshot.lifecycle != PartitionLifecycle::Serving { + continue; + } + match sample(partition, snapshot.ownership_epoch, limit).await { + Ok(samples) => { + observations.insert(*id, (snapshot.ownership_epoch, samples)); + } + Err(error) => warn!(partition_id_high = id.high, partition_id_low = id.low, %error, + "split observation failed; partition remains unsampled"), + } + } + if !completion.0.is_closed() { + completion.0.cached.store(Arc::new(observations)); + } + }); + drop(owned_partitions); + drop(runtime_handle); + drop(completion); } } impl LoadSampling { pub(super) fn samples(&self, id: Id128, epoch: u64) -> Samples { + if self.is_closed() { + return Vec::new(); + } self.cached .load() .get(&id) @@ -38,10 +88,33 @@ impl LoadSampling { .map_or_else(Vec::new, |(_, samples)| samples.clone()) } + pub(super) fn close(&self) { + self.state.fetch_or(CLOSED, Ordering::AcqRel); + } + + pub(super) async fn drain(&self) { + self.close(); + loop { + let completed = self.completed.notified(); + tokio::pin!(completed); + completed.as_mut().enable(); + if self.state.load(Ordering::Acquire) & RUNNING == 0 { + // Closed admission and completed payload teardown make this final. + self.cached.store(Arc::new(HashMap::new())); + return; + } + completed.await; + } + } + + fn is_closed(&self) -> bool { + self.state.load(Ordering::Acquire) & CLOSED != 0 + } + pub(super) fn launch(self: &Arc, partitions: Arc>, limit: usize) { if self - .running - .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .state + .compare_exchange(0, RUNNING, Ordering::AcqRel, Ordering::Acquire) .is_err() { return; @@ -50,25 +123,12 @@ impl LoadSampling { let runtime = tokio::runtime::Handle::current(); // Native page reads may block on cold storage. One owned job per service // isolates that delay and prevents repeated heartbeat ticks spawning work. - tokio::task::spawn_blocking(move || { - runtime.block_on(async move { - let mut observations = HashMap::new(); - for (id, partition) in partitions.iter() { - let snapshot = partition.snapshot(); - if snapshot.lifecycle != PartitionLifecycle::Serving { - continue; - } - match sample(partition, snapshot.ownership_epoch, limit).await { - Ok(samples) => { - observations.insert(*id, (snapshot.ownership_epoch, samples)); - } - Err(error) => warn!(partition_id_high = id.high, partition_id_low = id.low, %error, - "split observation failed; partition remains unsampled"), - } - } - job.0.cached.store(Arc::new(observations)); - }); - }); + let work = SamplingWork { + partitions, + runtime, + completion: job, + }; + tokio::task::spawn_blocking(move || work.run(limit)); } } diff --git a/app/crowdb-chunk-kv-server/src/serving/transition_runtime.rs b/app/crowdb-chunk-kv-server/src/serving/transition_runtime.rs index e09edb80c..f9ff02226 100644 --- a/app/crowdb-chunk-kv-server/src/serving/transition_runtime.rs +++ b/app/crowdb-chunk-kv-server/src/serving/transition_runtime.rs @@ -57,6 +57,9 @@ impl TransitionProcessor { mut revision: u64, ) -> Result<(), MonitorError> { let mut machine = TransferStateMachine::restore(transition)?; + self.executor + .release_excluded_source(machine.transition()) + .await?; if machine.transition().source.instance_id == self.instance_id && machine.transition().phase == TransferPhase::Planned { diff --git a/app/crowdb-chunk-kv-server/src/serving/worker.rs b/app/crowdb-chunk-kv-server/src/serving/worker.rs index 2a433c4a3..7d181f7e6 100644 --- a/app/crowdb-chunk-kv-server/src/serving/worker.rs +++ b/app/crowdb-chunk-kv-server/src/serving/worker.rs @@ -30,6 +30,28 @@ pub struct TransitionExecutor { } impl TransitionExecutor { + pub(super) async fn release_excluded_source( + &self, + transition: &TransferTransition, + ) -> Result<(), MonitorError> { + if transition.source.instance_id != self.instance_id || transition.release_proof.is_none() { + return Ok(()); + } + let Some(partition) = self.service.hosted_partition(transition.partition_id) else { + return Ok(()); + }; + // A restarted source can reopen the still-published old assignment + // before cutover. Release its exact old WAL so the target can take over; + // a later assignment of the same partition must remain untouched. + if partition.snapshot().ownership_epoch != transition.source_epoch { + return Ok(()); + } + partition + .suspend_for_shutdown(transition.source_epoch) + .await + .map_err(|error| plan_error(&error.to_string())) + } + /// Creates a worker bound to one server identity. /// /// # Errors diff --git a/app/crowdb-chunk-kv-server/src/storage.rs b/app/crowdb-chunk-kv-server/src/storage.rs index 239539f99..ea7606605 100644 --- a/app/crowdb-chunk-kv-server/src/storage.rs +++ b/app/crowdb-chunk-kv-server/src/storage.rs @@ -98,6 +98,10 @@ impl ChunkKvStorage { chunk_capacity_bytes: config.storage.stream_chunk_capacity_bytes, ..StreamConfig::default() }, + ChunkReadPolicy { + preferred_node_id: config.node_id, + ..ChunkReadPolicy::default() + }, ) .await } @@ -109,13 +113,14 @@ impl ChunkKvStorage { writer_lease_ms: u64, tree: TreeStorageGeometry, stream_config: StreamConfig, + read_policy: ChunkReadPolicy, ) -> Result { let streams = Arc::new( ProductionStreamRuntime::new_with_mirror_copies( Arc::clone(&kv), &chunk_io, writer_lease_ms, - ChunkReadPolicy::default(), + read_policy, stream_config, tree.mirror_copies, ) @@ -147,6 +152,7 @@ impl ChunkKvStorage { strip_capacity_bytes: 0, }, StreamConfig::default(), + ChunkReadPolicy::default(), ) .await } @@ -528,16 +534,22 @@ impl ChunkKvStorage { if !fresh { return recover_bootstrap_partition(config, page_store, stream).await; } - let partition = Partition::open_native( - partition_id, - range, - config.owner_epoch, - PartitionConfig::default(), - config.tree_id, - crowdb_tree_ffi::Config::default(), - page_store, - stream, - ) + let owner_epoch = config.owner_epoch; + let tree_id = config.tree_id; + let partition = tokio::task::spawn_blocking(move || { + Partition::open_native( + partition_id, + range, + owner_epoch, + PartitionConfig::default(), + tree_id, + crowdb_tree_ffi::Config::default(), + page_store, + stream, + ) + }) + .await + .map_err(|error| StorageRuntimeError::Partition(format!("bootstrap tree open: {error}")))? .map_err(|error| StorageRuntimeError::Partition(format!("bootstrap tree open: {error}")))?; initialize_bootstrap_root(&partition, config).await?; Ok(partition) @@ -593,10 +605,16 @@ impl ChunkKvStorage { "nonempty child WAL has no committed handoff proof", )); } - crowdb_chunk_kv::CrowdbPartitionTree::open(child.tree_id, &child.tree_config) - .map_err(|error| storage_plan_error(&error.to_string()))? - .unpin_generation(split_plan(transition).transition_id) - .map_err(|error| storage_plan_error(&error.to_string()))?; + let tree_id = child.tree_id; + let tree_config = child.tree_config.clone(); + let transition_id = split_plan(transition).transition_id; + tokio::task::spawn_blocking(move || { + crowdb_chunk_kv::CrowdbPartitionTree::open(tree_id, &tree_config)? + .unpin_generation(transition_id) + }) + .await + .map_err(|error| storage_plan_error(&error.to_string()))? + .map_err(|error| storage_plan_error(&error.to_string()))?; child }; let handoff = Arc::new(split_handoff::SplitHandoffCommit { diff --git a/app/crowdb-chunk-kv-server/src/storage/root_catalog.rs b/app/crowdb-chunk-kv-server/src/storage/root_catalog.rs index 6b56650fa..be867266b 100644 --- a/app/crowdb-chunk-kv-server/src/storage/root_catalog.rs +++ b/app/crowdb-chunk-kv-server/src/storage/root_catalog.rs @@ -160,7 +160,7 @@ impl KvRootCatalogStore { && entry.owner_epoch == prepared.owner_epoch && entry.artifact.tree_id == prepared.artifact.tree_id && entry.artifact.stream_name == prepared.artifact.stream_name - && entry.transition_id == prepared.transition_id + && same_publication_dependency(entry, prepared) && same_recovery_base(entry, prepared) && (entry.state == ChunkKvRangeCatalogPartitionState::Serving || (allow_catching_up @@ -523,6 +523,17 @@ fn decode_u64(value: &[u8]) -> Option { value.try_into().ok().map(u64::from_be_bytes) } +fn same_publication_dependency( + entry: &ChunkKvRangeCatalogEntry, + prepared: &ChunkKvRangeCatalogEntry, +) -> bool { + entry.transition_id == prepared.transition_id + || (entry.state == ChunkKvRangeCatalogPartitionState::Serving + && entry.transition_id.is_none() + && entry.artifact.tail_overlay.is_none() + && prepared.artifact.tail_overlay.is_none()) +} + fn same_recovery_base(entry: &ChunkKvRangeCatalogEntry, prepared: &ChunkKvRangeCatalogEntry) -> bool { match (&entry.artifact.tail_overlay, &prepared.artifact.tail_overlay) { (None, None) => true, diff --git a/app/crowdb-chunk-kv-server/tests/catalog_test.rs b/app/crowdb-chunk-kv-server/tests/catalog_test.rs index ddd9a64fa..245211ac9 100644 --- a/app/crowdb-chunk-kv-server/tests/catalog_test.rs +++ b/app/crowdb-chunk-kv-server/tests/catalog_test.rs @@ -13,6 +13,9 @@ use crowdb_protocol::chunk_kv::{ }; use crowdb_protocol::chunk_stream::StreamName; +#[path = "catalog_test/recovery_assignments.rs"] +mod recovery_assignments; + fn page(generation: u64, owner_epoch: u64) -> ChunkKvRangeCatalogPage { let mut page = ChunkKvRangeCatalogPage { generation, diff --git a/app/crowdb-chunk-kv-server/tests/catalog_test/recovery_assignments.rs b/app/crowdb-chunk-kv-server/tests/catalog_test/recovery_assignments.rs new file mode 100644 index 000000000..6664d38b7 --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/catalog_test/recovery_assignments.rs @@ -0,0 +1,72 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; + +#[tokio::test] +async fn completed_replay_survives_remote_assignment_changes_after_original_deadline() { + let publisher = ChunkKvRangeCatalogPublisher::new(Arc::new(MemoryChunkKvRangeCatalogStore::default())); + let mut first = page(1, 1); + first.entries[0].range.end = Some(b"m".to_vec()); + let mut remote = first.entries[0].clone(); + remote.partition_id.low = 2; + remote.owner.instance_id = 9; + remote.owner.rpc_endpoint = "127.0.0.1:9901".into(); + remote.range.start = b"m".to_vec(); + remote.range.end = None; + remote.artifact.tree_id = 2; + remote.artifact.stream_name.low = 4; + first.entries.push(remote); + first.seal().unwrap(); + publisher + .publish(head(1, None, &first), vec![first.clone()]) + .await + .unwrap(); + let mut calls = 0; + let recovered = publisher + .recover_current_for_instance(8, std::time::Instant::now(), |pages| { + calls += 1; + let publisher = &publisher; + async move { + let mut next = pages[0].clone(); + next.generation = 2; + next.entries[1].owner_epoch = 2; + next.seal().unwrap(); + publisher + .publish(head(2, Some(1), &next), vec![next]) + .await + .unwrap(); + Ok::<_, &'static str>(pages[0].entries[0].owner_epoch) + } + }) + .await + .unwrap() + .unwrap(); + assert_eq!(calls, 1, "unchanged local artifacts must not replay twice"); + assert_eq!(recovered.0.generation, 2); + assert_eq!(recovered.2, 1); +} + +#[tokio::test] +async fn changed_local_epoch_never_reuses_completed_replay_after_deadline() { + let publisher = ChunkKvRangeCatalogPublisher::new(Arc::new(MemoryChunkKvRangeCatalogStore::default())); + let first = page(1, 1); + publisher + .publish(head(1, None, &first), vec![first]) + .await + .unwrap(); + let result = publisher + .recover_current_for_instance(8, std::time::Instant::now(), |_| { + let publisher = &publisher; + async move { + let next = page(2, 2); + publisher + .publish(head(2, Some(1), &next), vec![next]) + .await + .unwrap(); + Ok::<_, &'static str>(1) + } + }) + .await; + assert_eq!(result.unwrap_err(), ChunkKvRangeCatalogError::GenerationConflict); +} diff --git a/app/crowdb-chunk-kv-server/tests/common/sampling_tree.rs b/app/crowdb-chunk-kv-server/tests/common/sampling_tree.rs new file mode 100644 index 000000000..b942a3f77 --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/common/sampling_tree.rs @@ -0,0 +1,147 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::{ + any::Any, + future::{poll_fn, Future}, + sync::atomic::{AtomicBool, AtomicUsize, Ordering}, + sync::Arc, + task::Poll, + time::Duration, +}; + +use crowdb_chunk_kv::{ + memory::MemoryPartitionTree, MutationOperation, PartitionRange, PartitionTree, Result, ScanEntry, + ValueRevision, +}; +use tokio::sync::Notify; + +pub struct TestSamplingTree { + memory: MemoryPartitionTree, + pub entered: Notify, + pub release: Notify, + pub finished: AtomicBool, + pub active: AtomicBool, + pub started: AtomicUsize, +} + +impl TestSamplingTree { + pub fn new() -> Self { + Self { + memory: MemoryPartitionTree::with_tree_id(71), + entered: Notify::new(), + release: Notify::new(), + finished: AtomicBool::new(false), + active: AtomicBool::new(false), + started: AtomicUsize::new(0), + } + } + + async fn hold_read(&self) { + let mut timer = std::pin::pin!(tokio::time::sleep(Duration::from_secs(30))); + let mut release = std::pin::pin!(self.release.notified()); + let mut announced = false; + poll_fn(|cx| { + // Keep a real registered timer in the production blocking job. + // Runtime shutdown wakes it; polling after shutdown must panic. + let timer = timer.as_mut().poll(cx); + let release = release.as_mut().poll(cx); + if timer.is_ready() || release.is_ready() { + return Poll::Ready(()); + } + if !announced { + announced = true; + self.entered.notify_one(); + } + Poll::Pending + }) + .await; + } +} + +#[async_trait::async_trait] +impl PartitionTree for TestSamplingTree { + fn as_any(&self) -> &dyn Any { + self + } + + fn tree_id(&self) -> u64 { + self.memory.tree_id() + } + + async fn get(&self, key: &[u8]) -> Result> { + self.memory.get(key).await + } + + async fn scan_forward( + &self, + start: Option<&[u8]>, + inclusive: bool, + end: Option<&[u8]>, + limit: usize, + bytes: usize, + ) -> Result<(Vec, bool)> { + self.started.fetch_add(1, Ordering::AcqRel); + self.active.store(true, Ordering::Release); + self.hold_read().await; + let result = self + .memory + .scan_forward(start, inclusive, end, limit, bytes) + .await; + self.active.store(false, Ordering::Release); + self.finished.store(true, Ordering::Release); + result + } + + async fn seek_reverse( + &self, + start: &[u8], + inclusive: bool, + begin: Option<&[u8]>, + ) -> Result> { + self.memory.seek_reverse(start, inclusive, begin).await + } + + async fn scan_reverse( + &self, + start: Option<&[u8]>, + begin: Option<&[u8]>, + limit: usize, + bytes: usize, + ) -> Result<(Vec, bool)> { + self.memory.scan_reverse(start, begin, limit, bytes).await + } + + async fn apply(&self, seq: u64, operation: &MutationOperation) -> Result<()> { + self.memory.apply(seq, operation).await + } + + async fn advance_noop(&self, seq: u64) -> Result<()> { + self.memory.advance_noop(seq).await + } + + async fn checkpoint(&self, offset: u64) -> Result<(u64, u64)> { + self.memory.checkpoint(offset).await + } + + async fn checkpoint_snapshot(&self, offset: u64) -> Result<(u64, u64, Arc)> { + self.memory.checkpoint_snapshot(offset).await + } + + async fn rebuild_range( + &self, + tree_id: u64, + range: &PartitionRange, + config: crowdb_tree_ffi::Config, + ) -> Result<(Arc, crowdb_tree_ffi::RangeRebuildStats)> { + self.memory.rebuild_range(tree_id, range, config).await + } + + fn checkpoint_state(&self) -> Result<(u64, u64)> { + self.memory.checkpoint_state() + } + + fn last_applied_seq(&self) -> u64 { + self.memory.last_applied_seq() + } +} diff --git a/app/crowdb-chunk-kv-server/tests/shutdown_runtime_test.rs b/app/crowdb-chunk-kv-server/tests/shutdown_runtime_test.rs new file mode 100644 index 000000000..7d8a4e22a --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/shutdown_runtime_test.rs @@ -0,0 +1,233 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/sampling_tree.rs"] +mod sampling_tree; +#[path = "common/split_storage.rs"] +mod split_storage; +#[path = "common/process.rs"] +mod test_process; + +use std::{ + future::{poll_fn, Future}, + process::Command, + sync::{atomic::Ordering, Arc, Barrier}, + task::Poll, + time::Duration, +}; + +use crowdb_chunk_kv::{Partition, PartitionId, PartitionLifecycle, PartitionRange}; +use crowdb_chunk_kv_server::ChunkKvService; +use sampling_tree::TestSamplingTree; +use split_storage::TestSplitStorage; + +const CHILD: &str = "CROWDB_TEST_SAMPLING_SHUTDOWN_CHILD"; + +#[test] +fn service_drain_owns_sampling_until_runtime_shutdown() { + isolated_child( + "service_drain_owns_sampling_until_runtime_shutdown", + run_shutdown_child, + ); +} + +fn isolated_child(name: &str, run: fn()) { + if std::env::var_os(CHILD).is_some() { + run(); + return; + } + let output = Command::new("pixi") + .arg("run") + .args(["timeout", "--signal=KILL", "20s"]) + .arg(std::env::current_exe().unwrap()) + .args(["--exact", name, "--nocapture"]) + .env(CHILD, "1") + .env("RUST_BACKTRACE", "full") + .output() + .expect("run isolated shutdown regression"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "shutdown child {name} failed: {}\nstdout:\n{stdout}\nstderr:\n{stderr}", + output.status + ); + assert!( + !stderr.contains("but it is being shutdown"), + "timer polled after shutdown:\n{stderr}" + ); +} + +fn run_shutdown_child() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .enable_all() + .build() + .unwrap(); + let escaped = runtime.block_on(async { + let storage = TestSplitStorage::new(); + let tree = Arc::new(TestSamplingTree::new()); + let (partition, _) = storage + .partition( + PartitionId { high: 71, low: 1 }, + PartitionRange::default(), + 1, + tree.clone(), + ) + .await; + let service = ChunkKvService::new(test_process::TestProcess::authority(1), 8).unwrap(); + service.install_partition(&partition).unwrap(); + service + .registry_observation_with_load_samples(4096, 0, 2) + .await + .unwrap(); + tree.entered.notified().await; + println!("production sampler registered its timer"); + + service.begin_drain(); + // Finish the real journal release first, so unrelated writer scheduling + // cannot make a WAL-only service drain look like a sampler barrier. + partition.suspend_for_shutdown(1).await.unwrap(); + let mut drain = std::pin::pin!(service.drain_writers()); + let first = poll_fn(|cx| Poll::Ready(drain.as_mut().poll(cx))).await; + match first { + Poll::Ready(result) => { + result.unwrap(); + assert!(!tree.finished.load(Ordering::Acquire)); + println!("service drain returned with the production sampling read still active"); + true + } + Poll::Pending => { + tree.release.notify_one(); + drain.await.unwrap(); + assert!(tree.finished.load(Ordering::Acquire)); + assert_eq!( + partition.snapshot().lifecycle, + PartitionLifecycle::TransferQuiesced + ); + false + } + } + }); + drop(runtime); + assert!( + !escaped, + "service drain completed before its sampling job; runtime shutdown overtook owned storage work" + ); +} + +fn sampling_service( + runtime: &tokio::runtime::Runtime, +) -> (Arc, Partition, Arc) { + runtime.block_on(async { + let storage = TestSplitStorage::new(); + let tree = Arc::new(TestSamplingTree::new()); + let (partition, _) = storage + .partition( + PartitionId { high: 71, low: 1 }, + PartitionRange::default(), + 1, + tree.clone(), + ) + .await; + let service = Arc::new(ChunkKvService::new(test_process::TestProcess::authority(1), 8).unwrap()); + service.install_partition(&partition).unwrap(); + (service, partition, tree) + }) +} + +#[test] +fn observation_after_drain_admission_cannot_launch_sampling() { + isolated_child( + "observation_after_drain_admission_cannot_launch_sampling", + run_closed_observation_child, + ); +} + +fn run_closed_observation_child() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .enable_all() + .build() + .unwrap(); + let (service, partition, tree) = sampling_service(&runtime); + // Keep the partition Serving: fencing its WAL first would let the old + // sampler skip it for an unrelated reason and conceal missing admission. + assert_eq!(partition.snapshot().lifecycle, PartitionLifecycle::Serving); + service.begin_drain(); + runtime + .block_on(service.registry_observation_with_load_samples(4096, 0, 2)) + .unwrap(); + drop(runtime); + assert_eq!(tree.started.load(Ordering::Acquire), 0); +} + +#[test] +fn concurrent_observation_and_close_cannot_escape_completed_drain() { + isolated_child( + "concurrent_observation_and_close_cannot_escape_completed_drain", + run_concurrent_close_child, + ); +} + +fn run_concurrent_close_child() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .enable_all() + .build() + .unwrap(); + let (service, partition, tree) = sampling_service(&runtime); + let barrier = Arc::new(Barrier::new(3)); + let observer = { + let service = service.clone(); + let barrier = barrier.clone(); + let handle = runtime.handle().clone(); + std::thread::spawn(move || { + let _entered = handle.enter(); + barrier.wait(); + handle + .block_on(service.registry_observation_with_load_samples(4096, 0, 2)) + .unwrap(); + }) + }; + let closer = { + let service = service.clone(); + let barrier = barrier.clone(); + std::thread::spawn(move || { + barrier.wait(); + service.begin_drain(); + }) + }; + barrier.wait(); + observer.join().unwrap(); + closer.join().unwrap(); + runtime.block_on(async { + // Remove WAL-release scheduling as a competing reason to be Pending. + partition.suspend_for_shutdown(1).await.unwrap(); + let mut drain = std::pin::pin!(service.drain_writers()); + if tree.active.load(Ordering::Acquire) { + let first = poll_fn(|cx| Poll::Ready(drain.as_mut().poll(cx))).await; + assert!(first.is_pending(), "drain escaped an active sampling read"); + } + // Also releases a read that passed lifecycle validation just before + // fencing but has not yet reached the injected storage boundary. + tree.release.notify_one(); + drain.await.unwrap(); + assert!(!tree.active.load(Ordering::Acquire)); + }); + drop(service); + drop(partition); + // The ordinary mutation worker also owns the tree until its closed + // receiver is polled. This is eventual fixture cleanup, not the sampler + // barrier oracle above. + runtime.block_on(async { + tokio::time::timeout(Duration::from_secs(1), async { + while Arc::strong_count(&tree) != 1 { + tokio::task::yield_now().await; + } + }) + .await + .expect("ordinary partition worker did not release the fixture tree"); + }); + drop(runtime); +} diff --git a/app/crowdb-chunk-kv-server/tests/transfer_storage_test.rs b/app/crowdb-chunk-kv-server/tests/transfer_storage_test.rs index 51089e6ac..ddadaca8d 100644 --- a/app/crowdb-chunk-kv-server/tests/transfer_storage_test.rs +++ b/app/crowdb-chunk-kv-server/tests/transfer_storage_test.rs @@ -163,3 +163,89 @@ async fn only_published_matching_target_can_claim_mutable_root() { .await .is_err()); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn released_recovery_marker_does_not_block_the_first_target_checkpoint() { + let metadata = metadata::TestMetadata::start().await; + let kv = metadata.client.clone(); + let source_entry = source(); + let source = open_root_catalog_for_tests(kv.clone(), 0, 1, &source_entry) + .await + .unwrap(); + source.publish(42, 0, 1, 1, b"source-root").unwrap(); + publish_assignment(kv.clone(), source_entry.clone(), 1).await; + + // Lease-excluded recovery adopts the original stream without an overlay. + let mut recovered_entry = source_entry; + recovered_entry.owner.instance_id = 2; + recovered_entry.owner_epoch = 2; + recovered_entry.state = ChunkKvRangeCatalogPartitionState::Prepared; + recovered_entry.transition_id = Some(Id128 { high: 5, low: 6 }); + let recovered = open_root_catalog_for_tests(kv.clone(), 0, 1, &recovered_entry) + .await + .unwrap(); + assert_eq!( + recovered.publish(42, 1, 2, 2, b"target-root"), + Err(CtError::Unavailable) + ); + + // The monitor may release the completed marker before the target's first + // checkpoint; the exact Serving assignment still establishes ownership. + recovered_entry.state = ChunkKvRangeCatalogPartitionState::Serving; + recovered_entry.transition_id = None; + publish_assignment(kv.clone(), recovered_entry.clone(), 2).await; + recovered.publish(42, 1, 2, 2, b"target-root").unwrap(); + assert_eq!( + source.publish(42, 2, 1, 3, b"stale-source"), + Err(CtError::Unavailable) + ); + let reopened = open_root_catalog_for_tests(kv, 0, 1, &recovered_entry) + .await + .unwrap(); + assert_eq!( + reopened.load(42, RootCatalogObject::CurrentManifest).unwrap(), + Some(b"target-root".to_vec()) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn released_marker_on_another_assignment_does_not_authorize_the_prepared_root() { + for mismatch in ["owner", "epoch", "tree", "stream"] { + let metadata = metadata::TestMetadata::start().await; + let kv = metadata.client.clone(); + let source_entry = source(); + let source = open_root_catalog_for_tests(kv.clone(), 0, 1, &source_entry) + .await + .unwrap(); + source.publish(42, 0, 1, 1, b"source-root").unwrap(); + publish_assignment(kv.clone(), source_entry.clone(), 1).await; + let mut prepared = source_entry; + prepared.owner.instance_id = 2; + prepared.owner_epoch = 2; + prepared.state = ChunkKvRangeCatalogPartitionState::Prepared; + prepared.transition_id = Some(Id128 { high: 5, low: 6 }); + let target = open_root_catalog_for_tests(kv.clone(), 0, 1, &prepared) + .await + .unwrap(); + let mut published = prepared; + published.state = ChunkKvRangeCatalogPartitionState::Serving; + published.transition_id = None; + match mismatch { + "owner" => published.owner.instance_id = 3, + "epoch" => published.owner_epoch = 3, + "tree" => published.artifact.tree_id = 43, + "stream" => published.artifact.stream_name.low = 3, + _ => unreachable!(), + } + publish_assignment(kv, published, 2).await; + assert_eq!( + target.publish(42, 1, 2, 2, b"unpublished-target"), + Err(CtError::Unavailable), + "released marker must not bypass {mismatch} identity" + ); + assert_eq!( + source.load(42, RootCatalogObject::CurrentManifest).unwrap(), + Some(b"source-root".to_vec()) + ); + } +} diff --git a/app/crowdb-chunk-kv-server/tests/transition_worker_test.rs b/app/crowdb-chunk-kv-server/tests/transition_worker_test.rs index fc33b2666..b0d89dbbe 100644 --- a/app/crowdb-chunk-kv-server/tests/transition_worker_test.rs +++ b/app/crowdb-chunk-kv-server/tests/transition_worker_test.rs @@ -313,6 +313,46 @@ fn transfer(phase: TransferPhase) -> TransferTransition { } } +#[tokio::test] +async fn returning_source_releases_its_writer_after_durable_lease_exclusion() { + for (epoch, prepared) in [(3, false), (3, true), (4, true)] { + let store = Arc::new(Group0ControlStore::new(Arc::new(MemoryKv::default()))); + let mut transition = transfer(TransferPhase::TargetPreparing); + transition.target_artifact = transition.artifact.clone(); + transition.release_proof = Some(AuthorityReleaseProof::LeaseExpired { + activation_not_before_ms: 105, + }); + store.persist_transfer_transition(&transition, 0).await.unwrap(); + let source = partition(id(1), KeyRange::default(), epoch, &artifact(11), prepared).await; + let service = Arc::new(ChunkKvService::new(test_process::TestProcess::authority(1), 4).unwrap()); + service.install_partition(&source).unwrap(); + let executor = Arc::new( + TransitionExecutor::with_storage( + 1, + service, + Arc::new(FakeStorage { + recovered: source.clone(), + split: split_artifact(&split_transition()), + expected_split_parent_range: None, + }), + 8, + ) + .unwrap(), + ); + let processor = TransitionProcessor::new(1, store, executor); + processor.tick().await.unwrap(); + processor.tick().await.unwrap(); + assert_eq!( + source.lifecycle(), + if epoch == 3 { + PartitionLifecycle::TransferQuiesced + } else { + PartitionLifecycle::Prepared + } + ); + } +} + fn split_transition() -> SplitTransition { SplitTransition { transition_id: id(91), diff --git a/app/crowdb-chunkdb/Cargo.toml b/app/crowdb-chunkdb/Cargo.toml index cacc1ccb0..a7b7bfffd 100644 --- a/app/crowdb-chunkdb/Cargo.toml +++ b/app/crowdb-chunkdb/Cargo.toml @@ -53,6 +53,7 @@ rand = "0.8" futures = "0.3" [dev-dependencies] +tokio = { workspace = true, features = ["test-util"] } crowdb-console-shared = { path = "../../lib/crowdb-console-shared" } crowdb-kv = { path = "../../lib/crowdb-kv", features = ["test-util"] } crowdb-chunkdb-client = { path = "../../lib/crowdb-chunkdb-client" } diff --git a/app/crowdb-chunkdb/src/allocator/pool.rs b/app/crowdb-chunkdb/src/allocator/pool.rs index bc25422a8..4a8e522f3 100644 --- a/app/crowdb-chunkdb/src/allocator/pool.rs +++ b/app/crowdb-chunkdb/src/allocator/pool.rs @@ -174,9 +174,10 @@ impl DiskdbClientPool { /// Allocate blocks on the diskdb instance owning `disk_group_id`. /// - /// A connection failure before submission refreshes discovery and permits - /// one attempt at a different endpoint. Once submitted, the mutation is - /// never replayed: DiskDB may have persisted blocks before losing its reply. + /// A connection failure before submission or an explicit ownership rejection + /// refreshes discovery and permits one attempt at a different endpoint. + /// Ambiguous submitted mutations are never replayed: DiskDB may have + /// persisted blocks before losing its reply. /// /// # Errors /// Returns `DiskdbClientError::ConnectFailed` when connection setup fails, @@ -240,7 +241,10 @@ impl DiskdbClientPool { DiskdbClientError::Unreachable(format!("no endpoint for disk_group {dg_id}: {error}")) })?; let result = self.transport.allocate_blocks(&endpoint, request).await; - if matches!(result, Err(DiskdbClientError::ConnectFailed(_))) { + if matches!( + result, + Err(DiskdbClientError::ConnectFailed(_) | DiskdbClientError::NotOwner(_)) + ) { if self.refresh_endpoints().await.is_err() { return result; } diff --git a/app/crowdb-chunkdb/src/lib.rs b/app/crowdb-chunkdb/src/lib.rs index 0278dd1a6..50d5f5396 100644 --- a/app/crowdb-chunkdb/src/lib.rs +++ b/app/crowdb-chunkdb/src/lib.rs @@ -21,6 +21,7 @@ pub mod metrics; pub mod placement_rebalance; pub mod placement_repair; pub mod range_guard; +pub mod readiness; pub mod relocation; pub mod repair; pub mod routing; diff --git a/app/crowdb-chunkdb/src/main.rs b/app/crowdb-chunkdb/src/main.rs index d9d6b7a9d..8a8741091 100644 --- a/app/crowdb-chunkdb/src/main.rs +++ b/app/crowdb-chunkdb/src/main.rs @@ -165,11 +165,11 @@ async fn main() { .parse() .expect("valid rpc_listen_addr"); - // Build KV client for group-0 topology access. + // Share discovery and metadata while keeping point reads off scan frames. let mut kv_config = ClientConfig::new(config.server.kv_server_mgmt_seeds.clone()); kv_config.pool_size_per_endpoint = config.server.kv_pool_size; kv_config.rpc_workers = config.server.kv_rpc_workers; - let kv = Arc::new(CrowdbKvClient::new(kv_config)); + let kv = Arc::new(CrowdbKvClient::new_with_isolated_point_reads(kv_config)); let hw = HardwareClient::from_shared(Arc::clone(&kv)); let refresh_hw = HardwareClient::from_shared(Arc::clone(&kv)); let watch = WatchNotifyClient::from_shared(Arc::clone(&kv)); @@ -325,35 +325,10 @@ async fn main() { }, ); - let range_refresh_guard = Arc::clone(&range_guard); - let range_refresh_kv = Arc::clone(&kv); - let range_refresh_stop = stop_rx.clone(); - let range_refresh_instance = config - .server - .instance_id - .as_ref() - .and_then(|value| value.parse::().ok()); - let range_refresh_handle = tokio::spawn(async move { - let Some(instance_id) = range_refresh_instance else { - return; - }; - let mut stop = range_refresh_stop; - let mut ticker = tokio::time::interval(Duration::from_secs(1)); - loop { - tokio::select! { - _ = ticker.tick() => { - if let Err(error) = range_refresh_guard.load_from_group0(&range_refresh_kv, instance_id).await { - warn!(%error, instance_id, "periodic range guard refresh failed"); - } - } - changed = stop.changed() => { - if changed.is_ok() && *stop.borrow() { - return; - } - } - } - } - }); + let readiness = Arc::new(crowdb_chunkdb::readiness::RangeReadiness::new(Arc::clone( + &range_guard, + ))); + let range_refresh_handle = Arc::clone(&readiness).start(Arc::clone(&kv), instance_id, stop_rx.clone()); // Diskdb client pool + chunk allocator. let pool = Arc::new(DiskdbClientPool::with_transport( @@ -454,15 +429,6 @@ async fn main() { info!(%rpc_listen_addr, "crowdb-rpc server listening"); // Start HTTP health + metrics + cache invalidation server. - let readiness = HttpReadiness { - range_guard: Arc::clone(&range_guard), - kv: Arc::clone(&kv), - instance_id: config - .server - .instance_id - .as_ref() - .and_then(|value| value.parse().ok()), - }; let http_handle = tokio::spawn(run_http_server( http_listen_addr, readiness, @@ -605,30 +571,17 @@ fn spawn_chunkdb_keepalive( Some(handle) } -struct HttpReadiness { - range_guard: Arc, - kv: Arc, - instance_id: Option, -} - /// HTTP server — health, metrics, cache invalidation endpoints. async fn run_http_server( addr: SocketAddr, - readiness: HttpReadiness, + readiness: Arc, locks: Arc, conversion_metrics: Arc, repair_metrics: Arc, conversion: [Arc; 2], ) { - let app = axum::Router::new() - .route( - "/ready", - axum::routing::get(move || { - let range_guard = Arc::clone(&readiness.range_guard); - let kv = Arc::clone(&readiness.kv); - async move { ready_response(&range_guard, &kv, readiness.instance_id).await } - }), - ) + let app = readiness + .router() .route("/health", axum::routing::get(|| async { "ok" })) .route( "/metrics", @@ -714,29 +667,6 @@ async fn run_http_server( axum::serve(listener, app).await.expect("HTTP server error"); } -async fn ready_response( - range_guard: &RangeGuard, - kv: &Arc, - instance_id: Option, -) -> (axum::http::StatusCode, &'static str) { - if let Some(instance_id) = instance_id { - if range_guard.load_from_group0(kv, instance_id).await.is_err() { - return ( - axum::http::StatusCode::SERVICE_UNAVAILABLE, - "range ownership refresh failed", - ); - } - } - if range_guard.is_ready() { - (axum::http::StatusCode::OK, "ok") - } else { - ( - axum::http::StatusCode::SERVICE_UNAVAILABLE, - "range ownership pending", - ) - } -} - fn load_config(args: &Cli) -> ChunkdbConfig { let config_path = &args.config; let mut config = diff --git a/app/crowdb-chunkdb/src/readiness.rs b/app/crowdb-chunkdb/src/readiness.rs new file mode 100644 index 000000000..fbe4320e3 --- /dev/null +++ b/app/crowdb-chunkdb/src/readiness.rs @@ -0,0 +1,142 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! HTTP readers observe bounded ownership evidence without driving authority reads. + +use crate::range_guard::RangeGuard; +use arc_swap::ArcSwapOption; +use axum::{http::StatusCode, routing::get, Router}; +use crowdb_kv_client::CrowdbKvClient; +use std::{future::Future, sync::Arc, time::Duration}; +use tokio::{ + sync::watch, + task::JoinHandle, + time::{Instant, MissedTickBehavior}, +}; + +const MAX_AGE: Duration = Duration::from_secs(3); + +struct Observation { + started: Instant, + succeeded: bool, +} + +pub struct RangeReadiness { + guard: Arc, + latest: ArcSwapOption, +} + +impl RangeReadiness { + pub fn new(guard: Arc) -> Self { + Self { + guard, + latest: ArcSwapOption::empty(), + } + } + + /// Use the same independently refreshed evidence for every HTTP reader. + pub fn router(self: Arc) -> Router { + Router::new().route( + "/ready", + get(move || { + let readiness = Arc::clone(&self); + async move { readiness.response() } + }), + ) + } + + fn response(&self) -> (StatusCode, &'static str) { + let Some(observed) = self.latest.load_full() else { + return ( + StatusCode::SERVICE_UNAVAILABLE, + "range ownership observation pending", + ); + }; + if observed.started.elapsed() > MAX_AGE { + return ( + StatusCode::SERVICE_UNAVAILABLE, + "range ownership observation expired", + ); + } + if !observed.succeeded { + return (StatusCode::SERVICE_UNAVAILABLE, "range ownership refresh failed"); + } + if self.guard.is_ready() { + (StatusCode::OK, "ok") + } else { + (StatusCode::SERVICE_UNAVAILABLE, "range ownership pending") + } + } + + /// Own one bounded refresh at a time; callers never initiate extra reads. + pub fn start( + self: Arc, + kv: Arc, + instance_id: u64, + stop: watch::Receiver, + ) -> JoinHandle<()> { + let guard = Arc::clone(&self.guard); + self.start_source(stop, move || { + let guard = Arc::clone(&guard); + let kv = Arc::clone(&kv); + async move { + guard + .load_from_group0(&kv, instance_id) + .await + .map_err(|error| error.to_string()) + } + }) + } + + fn start_source(self: Arc, mut stop: watch::Receiver, source: F) -> JoinHandle<()> + where + F: Fn() -> Fut + Send + 'static, + Fut: Future> + Send, + { + tokio::spawn(async move { + let mut ticker = tokio::time::interval(Duration::from_secs(1)); + ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + if *stop.borrow() { + return; + } + tokio::select! { + _ = ticker.tick() => {}, + _ = stop.changed() => return, + } + tokio::select! { + () = self.sample(source()) => {}, + _ = stop.changed() => return, + } + } + }) + } + + async fn sample(&self, source: impl Future>) { + let started = Instant::now(); + let result = tokio::time::timeout(MAX_AGE, source) + .await + .unwrap_or_else(|_| Err("range ownership refresh exceeded its three-second deadline".into())); + self.latest.store(Some(Arc::new(Observation { + started, + succeeded: result.is_ok(), + }))); + if let Err(error) = result { + tracing::warn!(%error, "periodic range guard refresh failed; readiness remains unavailable"); + } + } + + /// Exercise production scheduling, expiry and cancellation with deterministic I/O. + #[cfg(feature = "test-util")] + pub fn start_source_for_tests( + self: Arc, + stop: watch::Receiver, + source: F, + ) -> JoinHandle<()> + where + F: Fn() -> Fut + Send + 'static, + Fut: Future> + Send, + { + self.start_source(stop, source) + } +} diff --git a/app/crowdb-chunkdb/src/repair.rs b/app/crowdb-chunkdb/src/repair.rs index b10991abb..e94e98bf9 100644 --- a/app/crowdb-chunkdb/src/repair.rs +++ b/app/crowdb-chunkdb/src/repair.rs @@ -25,6 +25,9 @@ use crate::metrics::RepairMetrics; use crate::task::executor::TaskFuture; use crate::task::{TaskHandler, TaskManager, TaskOutcome, TaskStore, TaskStoreError}; +mod memory; +mod targets; + pub const REPAIR_STRIP_TASK_VERSION: u16 = 1; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -209,7 +212,6 @@ impl RepairStripTaskHandler { self } - #[allow(clippy::too_many_lines)] async fn execute_once(&self, task: &ChunkTaskValue) -> Result { let mut payload = decode_payload(&task.payload)?; let chunk = self.lifecycle.query_chunk(&payload.chunk_id).await?; @@ -224,6 +226,7 @@ impl RepairStripTaskHandler { let segments = strip_segments(strip)?; self.confirm_published_targets(task, &mut payload, &segments) .await?; + self.reconcile_allocated_targets(task, &mut payload).await?; let unavailable: Vec<_> = strip .unavailable_segments .iter() @@ -237,45 +240,9 @@ impl RepairStripTaskHandler { .checked_mul(1024) .ok_or_else(|| RepairRunError::Permanent("unit size overflows".into()))?; let shard_bytes = segment_size(&segments[0], unit_bytes)?; - let required = shard_bytes - .checked_mul(u64::try_from(segments.len().saturating_add(1)).unwrap_or(u64::MAX)) - .and_then(|bytes| usize::try_from(bytes).ok()) - .ok_or_else(|| RepairRunError::Permanent("repair memory calculation overflows".into()))?; - if required > self.memory_limit { - return Err(RepairRunError::Retry(format!( - "repair requires {required} bytes but the limit is {}", - self.memory_limit - ))); - } - let permits = u32::try_from(required) - .map_err(|_| RepairRunError::Permanent("repair memory exceeds semaphore".into()))?; - let _reservation = self - .memory - .clone() - .acquire_many_owned(permits) - .await - .map_err(|_| RepairRunError::Retry("repair memory budget closed".into()))?; - self.metrics.reserve_memory(required); - let _memory_metrics = RepairMemoryGuard { - metrics: Arc::clone(&self.metrics), - bytes: required, - }; + let (_reservation, _memory_metrics) = self.reserve_memory(shard_bytes, segments.len()).await?; - let recovered = match strip.strip.as_ref() { - Some(Strip::MirrorStrip(_)) => self.recover_mirror(strip, &segments).await?, - Some(Strip::EcStrip(ec)) => { - let sealed_bytes = if chunk.acknowledged_cursor == 0 { - u64::from(strip.sealed_length) * 1024 - } else { - chunk - .acknowledged_cursor - .saturating_sub(u64::from(strip.chunk_offset) * 1024) - .min(u64::from(strip.capacity) * 1024) - }; - self.recover_ec(strip, ec, &segments, sealed_bytes).await? - } - None => return Err(RepairRunError::Permanent("strip has no body".into())), - }; + let recovered = self.recover_shards(&chunk, strip, &segments).await?; if !recovered.new_failures.is_empty() { self.persist_new_failures( &chunk, @@ -288,94 +255,9 @@ impl RepairStripTaskHandler { return Err(RepairRunError::Retry("new failed segments were persisted".into())); } - let mut replacement = strip.clone(); - let mut surviving: Vec<_> = segments - .iter() - .copied() - .filter(|segment| !unavailable.contains(segment)) - .collect(); - // Even when node anti-affinity is relaxed for an undersized test - // cluster, never place two shards from one strip on the same disk. - let mut excluded: Vec<_> = segments.iter().filter_map(|segment| segment.disk_id).collect(); - for failed in &unavailable { - let index = segments - .iter() - .position(|segment| segment == failed) - .ok_or_else(|| RepairRunError::Permanent("failed segment disappeared".into()))?; - let target_index = payload.targets.iter().position(|target| target.source == *failed); - let target_index = if let Some(index) = target_index { - index - } else { - let destination = self - .lifecycle - .allocate_repair_segment( - &payload.chunk_id, - failed, - &surviving, - &excluded, - self.allow_unsafe_placement, - ) - .await?; - payload.targets.push(RepairTargetCheckpoint { - source: *failed, - destination, - phase: RepairTargetPhase::Allocated, - }); - self.checkpoint(task, &payload).await?; - payload.targets.len().saturating_sub(1) - }; - let destination = payload.targets[target_index].destination; - if payload.targets[target_index].phase == RepairTargetPhase::Allocated { - let mut data = recovered.shards[index].clone(); - if let Some(Strip::EcStrip(ec)) = &strip.strip { - let sealed = if chunk.acknowledged_cursor == 0 { - u64::from(strip.sealed_length) * 1024 - } else { - chunk - .acknowledged_cursor - .saturating_sub(u64::from(strip.chunk_offset) * 1024) - .min(u64::from(strip.capacity) * 1024) - }; - let valid = if index < ec.data_num as usize { - sealed.saturating_sub(index as u64 * shard_bytes).min(shard_bytes) - } else { - sealed.min(shard_bytes) - }; - data = data.slice(..usize::try_from(valid).unwrap_or(data.len())); - } - if !data.is_empty() { - self.io - .write_segment(&destination, unit_bytes, data) - .await - .map_err(|error| RepairRunError::Retry(error.to_string()))?; - } - self.io - .fsync_segment(&destination) - .await - .map_err(|error| RepairRunError::Retry(error.to_string()))?; - payload.targets[target_index].phase = RepairTargetPhase::Copied; - self.checkpoint(task, &payload).await?; - } - if let Some(shared) = &self.ad_hoc { - shared.rebuilt( - RecoveryKey { - chunk_id: payload.chunk_id, - strip_sequence: payload.strip_sequence, - segment: *failed, - }, - recovered.shards[index].clone(), - ); - } - let new_segment = destination; - replace_segment(&mut replacement, failed, new_segment)?; - surviving.push(new_segment); - if let Some(disk_id) = new_segment.disk_id { - excluded.push(disk_id); - } - } - replacement - .unavailable_segments - .retain(|segment| !unavailable.contains(segment)); + let replacement = self + .prepare_replacement(task, &mut payload, &chunk, strip, &recovered, &unavailable) + .await?; self.lifecycle .publish_tentative_chunk_strip_range( &payload.chunk_id, @@ -459,6 +341,29 @@ impl RepairStripTaskHandler { self.checkpoint(task, payload).await } + async fn recover_shards( + &self, + chunk: &Chunk, + strip: &ChunkStrip, + segments: &[Segment], + ) -> Result { + match strip.strip.as_ref() { + Some(Strip::MirrorStrip(_)) => self.recover_mirror(strip, segments).await, + Some(Strip::EcStrip(ec)) => { + let sealed_bytes = if chunk.acknowledged_cursor == 0 { + u64::from(strip.sealed_length) * 1024 + } else { + chunk + .acknowledged_cursor + .saturating_sub(u64::from(strip.chunk_offset) * 1024) + .min(u64::from(strip.capacity) * 1024) + }; + self.recover_ec(strip, ec, segments, sealed_bytes).await + } + None => Err(RepairRunError::Permanent("strip has no body".into())), + } + } + async fn recover_mirror( &self, strip: &ChunkStrip, @@ -644,17 +549,6 @@ struct RepairStats { bytes: u64, } -struct RepairMemoryGuard { - metrics: Arc, - bytes: usize, -} - -impl Drop for RepairMemoryGuard { - fn drop(&mut self) { - self.metrics.release_memory(self.bytes); - } -} - #[derive(Debug, thiserror::Error)] enum RepairRunError { #[error(transparent)] diff --git a/app/crowdb-chunkdb/src/repair/memory.rs b/app/crowdb-chunkdb/src/repair/memory.rs new file mode 100644 index 000000000..b668d1c0c --- /dev/null +++ b/app/crowdb-chunkdb/src/repair/memory.rs @@ -0,0 +1,54 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Memory admission for one full-segment repair attempt. + +use super::{RepairMetrics, RepairRunError, RepairStripTaskHandler}; +use std::sync::Arc; +use tokio::sync::OwnedSemaphorePermit; + +pub(super) struct RepairMemoryGuard { + metrics: Arc, + bytes: usize, +} + +impl Drop for RepairMemoryGuard { + fn drop(&mut self) { + self.metrics.release_memory(self.bytes); + } +} + +impl RepairStripTaskHandler { + pub(super) async fn reserve_memory( + &self, + shard_bytes: u64, + segment_count: usize, + ) -> Result<(OwnedSemaphorePermit, RepairMemoryGuard), RepairRunError> { + let required = shard_bytes + .checked_mul(u64::try_from(segment_count.saturating_add(1)).unwrap_or(u64::MAX)) + .and_then(|bytes| usize::try_from(bytes).ok()) + .ok_or_else(|| RepairRunError::Permanent("repair memory calculation overflows".into()))?; + if required > self.memory_limit { + return Err(RepairRunError::Retry(format!( + "repair requires {required} bytes but the limit is {}", + self.memory_limit, + ))); + } + let permits = u32::try_from(required) + .map_err(|_| RepairRunError::Permanent("repair memory exceeds semaphore".into()))?; + let reservation = self + .memory + .clone() + .acquire_many_owned(permits) + .await + .map_err(|_| RepairRunError::Retry("repair memory budget closed".into()))?; + self.metrics.reserve_memory(required); + Ok(( + reservation, + RepairMemoryGuard { + metrics: Arc::clone(&self.metrics), + bytes: required, + }, + )) + } +} diff --git a/app/crowdb-chunkdb/src/repair/targets.rs b/app/crowdb-chunkdb/src/repair/targets.rs new file mode 100644 index 000000000..007563e02 --- /dev/null +++ b/app/crowdb-chunkdb/src/repair/targets.rs @@ -0,0 +1,186 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Checkpointed tentative repair destinations and replacement assembly. + +use super::{replace_segment, segment_size, strip_segments, RecoveredShards, RepairRunError}; +use super::{RepairStripTaskHandler, RepairStripTaskV1}; +use crate::ad_hoc::RecoveryKey; +use bytes::Bytes; +use crowdb_protocol::chunk_task::{ChunkTaskValue, RepairTargetCheckpoint, RepairTargetPhase}; +use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkStrip, Strip}; +use crowdb_protocol::common::DiskId; +use crowdb_protocol::diskdb::rpc::Segment; + +impl RepairStripTaskHandler { + pub(super) async fn reconcile_allocated_targets( + &self, + task: &ChunkTaskValue, + payload: &mut RepairStripTaskV1, + ) -> Result<(), RepairRunError> { + if !payload + .targets + .iter() + .any(|target| target.phase == RepairTargetPhase::Allocated) + { + return Ok(()); + } + if payload.targets.iter().any(|target| { + target.phase == RepairTargetPhase::Allocated && target.destination.disk_id.is_none() + }) { + return Err(RepairRunError::Permanent( + "repair destination has no disk identity".into(), + )); + } + let snapshot = self.lifecycle.topology_snapshot(); + if snapshot.is_empty() { + return Err(RepairRunError::Retry( + "repair target topology is unavailable".into(), + )); + } + let healthy = snapshot.healthy_disk_groups(); + let previous = payload.targets.len(); + payload.targets.retain(|target| { + target.phase != RepairTargetPhase::Allocated + || target + .destination + .disk_id + .is_some_and(|disk| healthy.iter().any(|group| group.value.disk_ids.contains(&disk))) + }); + if payload.targets.len() != previous { + // Publish removal before allocating another destination. The exact + // old tentative block remains in DiskDB; its owner scanner applies + // the normal persisted absence grace, including after reply loss. + self.checkpoint(task, payload).await?; + } + Ok(()) + } + + pub(super) async fn prepare_replacement( + &self, + task: &ChunkTaskValue, + payload: &mut RepairStripTaskV1, + chunk: &Chunk, + strip: &ChunkStrip, + recovered: &RecoveredShards, + unavailable: &[Segment], + ) -> Result { + let segments = strip_segments(strip)?; + let unit_bytes = u64::from(strip.unit_kb) * 1024; + let shard_bytes = segment_size(&segments[0], unit_bytes)?; + let mut replacement = strip.clone(); + let mut surviving: Vec<_> = segments + .iter() + .copied() + .filter(|segment| !unavailable.contains(segment)) + .collect(); + // Preserve disk separation even when the node policy is relaxed. + let mut excluded: Vec<_> = segments.iter().filter_map(|segment| segment.disk_id).collect(); + for failed in unavailable { + let index = segments + .iter() + .position(|segment| segment == failed) + .ok_or_else(|| RepairRunError::Permanent("failed segment disappeared".into()))?; + let target = self + .target_index(task, payload, failed, &surviving, &excluded) + .await?; + let destination = payload.targets[target].destination; + if payload.targets[target].phase == RepairTargetPhase::Allocated { + let data = shard_data(chunk, strip, index, shard_bytes, recovered.shards[index].clone()); + self.copy_target(task, payload, target, unit_bytes, data).await?; + } + if let Some(shared) = &self.ad_hoc { + shared.rebuilt( + RecoveryKey { + chunk_id: payload.chunk_id, + strip_sequence: payload.strip_sequence, + segment: *failed, + }, + recovered.shards[index].clone(), + ); + } + replace_segment(&mut replacement, failed, destination)?; + surviving.push(destination); + if let Some(disk_id) = destination.disk_id { + excluded.push(disk_id); + } + } + replacement + .unavailable_segments + .retain(|segment| !unavailable.contains(segment)); + Ok(replacement) + } + + async fn target_index( + &self, + task: &ChunkTaskValue, + payload: &mut RepairStripTaskV1, + failed: &Segment, + surviving: &[Segment], + excluded: &[DiskId], + ) -> Result { + if let Some(index) = payload.targets.iter().position(|target| target.source == *failed) { + return Ok(index); + } + let destination = self + .lifecycle + .allocate_repair_segment( + &payload.chunk_id, + failed, + surviving, + excluded, + self.allow_unsafe_placement, + ) + .await?; + payload.targets.push(RepairTargetCheckpoint { + source: *failed, + destination, + phase: RepairTargetPhase::Allocated, + }); + self.checkpoint(task, payload).await?; + Ok(payload.targets.len().saturating_sub(1)) + } + + async fn copy_target( + &self, + task: &ChunkTaskValue, + payload: &mut RepairStripTaskV1, + index: usize, + unit_bytes: u64, + data: Bytes, + ) -> Result<(), RepairRunError> { + let destination = payload.targets[index].destination; + if !data.is_empty() { + self.io + .write_segment(&destination, unit_bytes, data) + .await + .map_err(|error| RepairRunError::Retry(error.to_string()))?; + } + self.io + .fsync_segment(&destination) + .await + .map_err(|error| RepairRunError::Retry(error.to_string()))?; + payload.targets[index].phase = RepairTargetPhase::Copied; + self.checkpoint(task, payload).await + } +} + +fn shard_data(chunk: &Chunk, strip: &ChunkStrip, index: usize, shard_bytes: u64, data: Bytes) -> Bytes { + let Some(Strip::EcStrip(ec)) = &strip.strip else { + return data; + }; + let sealed = if chunk.acknowledged_cursor == 0 { + u64::from(strip.sealed_length) * 1024 + } else { + chunk + .acknowledged_cursor + .saturating_sub(u64::from(strip.chunk_offset) * 1024) + .min(u64::from(strip.capacity) * 1024) + }; + let valid = if index < ec.data_num as usize { + sealed.saturating_sub(index as u64 * shard_bytes).min(shard_bytes) + } else { + sealed.min(shard_bytes) + }; + data.slice(..usize::try_from(valid).unwrap_or(data.len())) +} diff --git a/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/mutations.rs b/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/mutations.rs index d52c5fea9..f3fee7ee9 100644 --- a/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/mutations.rs +++ b/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/mutations.rs @@ -617,6 +617,7 @@ impl ChunkdbRpcService { create_nano, msg_type, result, + false, ); }); } @@ -714,6 +715,7 @@ impl ChunkdbRpcService { create_nano, msg_type, result, + true, ); }); } @@ -782,6 +784,7 @@ impl ChunkdbRpcService { create_nano, msg_type, result, + false, ); }); } @@ -924,6 +927,7 @@ impl ChunkdbRpcService { create_nano, msg_type, result, + false, ); }); } @@ -984,6 +988,7 @@ impl ChunkdbRpcService { create_nano, msg_type, result, + false, ); }); } diff --git a/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/wire.rs b/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/wire.rs index d42cd699e..8ef19d8c6 100644 --- a/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/wire.rs +++ b/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/wire.rs @@ -24,6 +24,13 @@ use crowdb_protocol::chunkdb_fb::{FBPlacementAssessment, FBPlacementAssessmentAr /// Map a `LifecycleError` to `(ret_code, message, range_start, range_end)`. pub(super) fn map_error(e: &LifecycleError) -> (FBChunkdbRetCode, String, u32, u32) { + map_error_with_policy(e, false) +} + +fn map_error_with_policy( + e: &LifecycleError, + allocation_retryable: bool, +) -> (FBChunkdbRetCode, String, u32, u32) { match e { LifecycleError::Storage(crate::storage::StoreError::Authority) => { (FBChunkdbRetCode::NotMyRange, e.to_string(), 0, 0) @@ -45,6 +52,19 @@ pub(super) fn map_error(e: &LifecycleError) -> (FBChunkdbRetCode, String, u32, u LifecycleError::Allocation(_) | LifecycleError::Commit(_) | LifecycleError::Cleanup(_) => { (FBChunkdbRetCode::Internal, e.to_string(), 0, 0) } + // Chunk creation rolls back its physical strip allocation when the + // metadata batch cannot be committed. A routed KV quorum/election + // outage is therefore safe for the client to retry as a fresh + // allocation; keep deterministic metadata failures internal. + LifecycleError::Storage(crate::storage::StoreError::Kv(message)) + if allocation_retryable + && (message.contains("leadership quorum unavailable") + || message.contains("retries exhausted") + || message.contains("no known leader") + || message.contains("transport error")) => + { + (FBChunkdbRetCode::Unavailable, e.to_string(), 0, 0) + } LifecycleError::Storage(_) => (FBChunkdbRetCode::Internal, e.to_string(), 0, 0), LifecycleError::InvalidRequest(_) => (FBChunkdbRetCode::InvalidArgument, e.to_string(), 0, 0), LifecycleError::LockBusy @@ -89,6 +109,7 @@ pub(super) fn submit_chunk_result( create_nano: u64, msg_type: u16, result: Result, + allocation_retryable: bool, ) { match result { Ok(chunk) => { @@ -97,7 +118,7 @@ pub(super) fn submit_chunk_result( submit_fb_response(server, conn_handle, ctrl, msg_type, req_id); } Err(e) => { - let (code, msg, rs, re) = map_error(&e); + let (code, msg, rs, re) = map_error_with_policy(&e, allocation_retryable); let ctrl = build_chunk_response(req_id, create_nano, code, Some(&msg), rs, re, &Chunk::default()); submit_fb_response(server, conn_handle, ctrl, msg_type, req_id); } @@ -1020,3 +1041,35 @@ pub(super) fn fb_ec_state(s: ProtoEcState) -> FBEcState { ProtoEcState::Parity => FBEcState::Parity, } } + +#[cfg(test)] +mod tests { + use super::{map_error, map_error_with_policy, FBChunkdbRetCode, LifecycleError}; + use crate::storage::StoreError; + + #[test] + fn transient_kv_quorum_errors_are_retryable() { + for message in [ + "linearizable read: leadership quorum unavailable", + "retries exhausted after 11 attempts", + "no known leader for group", + "transport error to endpoint", + ] { + let error = LifecycleError::Storage(StoreError::Kv(message.into())); + assert_eq!( + map_error_with_policy(&error, true).0, + FBChunkdbRetCode::Unavailable, + "{message}" + ); + assert_eq!(map_error(&error).0, FBChunkdbRetCode::Internal); + } + } + + #[test] + fn deterministic_kv_errors_remain_internal() { + let error = LifecycleError::Storage(StoreError::Kv("invalid request encoding".into())); + assert_eq!(map_error(&error).0, FBChunkdbRetCode::Internal); + let unknown = LifecycleError::Storage(StoreError::OutcomeUnknown("quorum unavailable".into())); + assert_eq!(map_error(&unknown).0, FBChunkdbRetCode::Internal); + } +} diff --git a/app/crowdb-chunkdb/src/storage.rs b/app/crowdb-chunkdb/src/storage.rs index 7fa124866..80f1f17a7 100644 --- a/app/crowdb-chunkdb/src/storage.rs +++ b/app/crowdb-chunkdb/src/storage.rs @@ -42,6 +42,8 @@ pub enum StoreError { Route(#[from] crate::routing::RouteError), #[error("kv client error: {0}")] Kv(String), + #[error("kv write outcome is unknown: {0}")] + OutcomeUnknown(String), #[error("serialization error: {0}")] Serde(String), #[error("chunk is outside this runtime domain or service slot authority")] @@ -117,7 +119,7 @@ impl ChunkStore { Err(crowdb_kv_client::Error::OutcomeUnknown) => match self.get_chunk(id).await { Ok(stored) if stored == *chunk => Ok(()), Ok(_) | Err(StoreError::ChunkNotFound) => Err(StoreError::Conflict), - Err(error) => Err(error), + Err(error) => Err(StoreError::OutcomeUnknown(error.to_string())), }, Err(crowdb_kv_client::Error::CasFailed { .. } | crowdb_kv_client::Error::CasBusy) => { Err(StoreError::ChunkAlreadyExists) diff --git a/app/crowdb-chunkdb/src/storage/scope.rs b/app/crowdb-chunkdb/src/storage/scope.rs index 864b1364f..abcd1f2a0 100644 --- a/app/crowdb-chunkdb/src/storage/scope.rs +++ b/app/crowdb-chunkdb/src/storage/scope.rs @@ -48,7 +48,8 @@ impl ChunkStore { } /// Keys begin with the owning chunk ID immediately after the prefix. - /// Continue past foreign slots until the requested owned result is filled. + /// Exclude other maintenance domains at KV, then continue past foreign + /// slots until the requested owned result is filled. pub(super) async fn scan_owned_records( &self, prefix: &[u8], @@ -58,6 +59,10 @@ impl ChunkStore { if max_keys == 0 || self.scope.as_ref().is_some_and(|(guard, _)| guard.is_empty()) { return Ok(Vec::new()); } + let (start, end) = self.scan_domain_bounds(prefix, start_after); + if !end.is_empty() && start.as_ref() >= end.as_slice() { + return Ok(Vec::new()); + } let table = self.bindings.snapshot(); if table.is_empty() { return Err(crate::routing::RouteError::NoBinding.into()); @@ -65,7 +70,7 @@ impl ChunkStore { let limit = usize::try_from(max_keys).unwrap_or(usize::MAX); let mut selected = Vec::new(); for route in table.bindings() { - let mut cursor = Bytes::copy_from_slice(start_after); + let mut cursor = start.clone(); let mut cutoff = 0; let mut matched = 0; loop { @@ -76,7 +81,7 @@ impl ChunkStore { route.kv_group_id, prefix, &cursor, - &[], + &end, 256, false, None, @@ -112,6 +117,23 @@ impl ChunkStore { } Ok(selected) } + + fn scan_domain_bounds(&self, prefix: &[u8], start_after: &[u8]) -> (Bytes, Vec) { + let Some((_, domain)) = &self.scope else { + return (Bytes::copy_from_slice(start_after), Vec::new()); + }; + // The purpose byte leads the big-endian ChunkId in both metadata + // namespaces. A short lower bound still includes its zero-valued ID. + let (first, end) = match domain { + ChunkDomain::System => (1, 4), + ChunkDomain::UserData => (4, 8), + }; + let mut lower = prefix.to_vec(); + lower.push(first); + let mut upper = prefix.to_vec(); + upper.push(end); + (Bytes::copy_from_slice(start_after.max(lower.as_slice())), upper) + } } fn key_chunk_id(key: &[u8], offset: usize) -> Result { diff --git a/app/crowdb-chunkdb/src/topology.rs b/app/crowdb-chunkdb/src/topology.rs index c97fbd9bb..bb11af798 100644 --- a/app/crowdb-chunkdb/src/topology.rs +++ b/app/crowdb-chunkdb/src/topology.rs @@ -3,9 +3,9 @@ //! Topology cache for chunkdb placement decisions. //! -//! `TopologyCache` holds an `Arc>` with the +//! `TopologyCache` publishes an immutable `ArcSwap` with the //! current cluster hierarchy (racks, nodes, disk-groups). Placement -//! (R87) calls `snapshot()` to get a consistent point-in-time view. +//! calls `snapshot()` to get a consistent point-in-time view. //! //! Update path: periodic full refresh from group-0 via //! `HardwareClient` + watch/notify for immediate status changes. @@ -23,6 +23,7 @@ use crowdb_protocol::{DiskGroupId, NodeId, RackId}; use crate::selector::PlacementEntry; +mod availability; pub mod notify; pub mod refresh; @@ -33,7 +34,7 @@ pub const G0_GROUP: u64 = 0; /// Watch prefixes for chunkdb topology updates. Disk records carry the /// allocation geometry; without their notifications a ChunkDB started before /// DiskIO publishes hardware can retain a zero unit size indefinitely. -pub const CHUNKDB_WATCH_PREFIXES: &[&[u8]] = &[b"/hw/node/", b"/hw/dg/", b"/hw/disk/"]; +pub const CHUNKDB_WATCH_PREFIXES: &[&[u8]] = &[b"/hw/node/", b"/hw/dg/", b"/hw/disk/", b"/srv/diskio/"]; /// `HwStatus::Up` as `i32` (prost represents enums as i32 in messages). const HW_UP: i32 = HwStatus::Up as i32; @@ -52,6 +53,8 @@ pub struct TopologySnapshot { disk_groups: HashMap, /// disk_id → physical failure-domain location for post-allocation checks. disks: HashMap, + /// Located DiskIO observations expire independently of hardware intent. + diskio_deadlines: HashMap, /// Capacity observations joined to live disk-group membership. usage: HashMap, /// Monotonic local publication generation. @@ -139,6 +142,16 @@ impl Drop for InFlightReservation { impl TopologySnapshot { /// All disk-groups with healthy status (rack/node/dg all `Up`). pub fn healthy_disk_groups(&self) -> Vec<&DiskGroupEntry> { + self.healthy_disk_groups_at(availability::now_ms()) + } + + #[cfg(feature = "test-util")] + #[must_use] + pub fn healthy_disk_groups_at_for_tests(&self, now: u64) -> Vec<&DiskGroupEntry> { + self.healthy_disk_groups_at(now) + } + + fn healthy_disk_groups_at(&self, now: u64) -> Vec<&DiskGroupEntry> { self.disk_groups .values() .filter(|dg| { @@ -148,7 +161,13 @@ impl TopologySnapshot { .get(&(dg.rack_id, dg.node_id)) .is_some_and(|(s, _)| *s == HW_UP); let rack_ok = self.racks.get(&dg.rack_id).is_some_and(|(s, _)| *s == HW_UP); - dg_ok && node_ok && rack_ok + dg_ok + && node_ok + && rack_ok + && self + .diskio_deadlines + .get(&dg.dg_id) + .map_or(true, |deadline| *deadline >= now) }) .collect() } @@ -497,6 +516,7 @@ pub async fn build_snapshot(hw: &crowdb_kv_client::HardwareClient) -> Option Option +// Licensed under the Apache License, Version 2.0. + +//! Physical IO availability overlays durable hardware intent for placement. + +use crowdb_protocol::{common::InstanceValue, sysdata::DiskGroupEntry, DiskGroupId}; +use std::{ + collections::HashMap, + time::{SystemTime, UNIX_EPOCH}, +}; + +pub(super) async fn read( + service: &crowdb_kv_client::ServiceRegistryClient, +) -> Option> { + match service.read_all_instance_observations("diskio").await { + Ok(instances) => Some(instances), + Err(error) => { + tracing::warn!(%error, "topology refresh: DiskIO availability is unavailable"); + None + } + } +} + +pub(super) fn now_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |time| u64::try_from(time.as_millis()).unwrap_or(u64::MAX)) +} + +pub(super) fn deadlines( + groups: &HashMap, + instances: &[(u64, InstanceValue)], +) -> HashMap { + let mut deadlines = HashMap::::new(); + for (_, instance) in instances { + let Some(extra) = instance.extra.as_ref().and_then(|extra| extra.diskdb.as_ref()) else { + continue; + }; + let (Some(rack), Some(node)) = (extra.rack_id, extra.node_id) else { + continue; + }; + for group in groups + .values() + .filter(|group| group.rack_id == rack && group.node_id == node) + { + if extra + .diskio_scope + .as_ref() + .is_some_and(|scope| scope.disk_group_id != 0 && scope.disk_group_id != group.dg_id) + { + continue; + } + let deadline = if extra.owned_dg_ids.contains(&group.dg_id) { + instance + .last_heartbeat_ms + .saturating_add(crowdb_kv_client::SERVICE_INSTANCE_TTL_MS) + } else { + 0 + }; + deadlines + .entry(group.dg_id) + .and_modify(|current| *current = (*current).max(deadline)) + .or_insert(deadline); + } + } + deadlines +} diff --git a/app/crowdb-chunkdb/tests/allocation_retry_test.rs b/app/crowdb-chunkdb/tests/allocation_retry_test.rs new file mode 100644 index 000000000..9eb726321 --- /dev/null +++ b/app/crowdb-chunkdb/tests/allocation_retry_test.rs @@ -0,0 +1,130 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/slot_groups.rs"] +#[allow(dead_code)] +mod slot_groups; + +use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbClientError, ChunkdbRpcTransport}; +use crowdb_kv_client::{ChunkSlotMapClient, ServiceRegistryClient}; +use crowdb_protocol::{ + chunk_slot::{ChunkSlotBootstrap, ChunkStorageGroup}, + chunkdb::rpc::{AllocateChunkRequest, ChunkType}, + chunkdb_fb::{ + FBAllocateChunkRequest, FBAllocateChunkResponse, FBAllocateChunkResponseArgs, FBChunk, FBChunkArgs, + FBChunkType, FBChunkdbRetCode, + }, + common::ChunkId, + fb::FBMsgType, +}; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +async fn client(test: &slot_groups::TestGroups, port: i32) -> ChunkdbClient { + let maps = ChunkSlotMapClient::new(Arc::clone(&test.kv)); + maps.initialize_layout(&ChunkSlotBootstrap { + service_instances: vec![1], + storage_groups: vec![ChunkStorageGroup { + store_id: 0, + group_id: 1, + }], + }) + .await + .unwrap(); + maps.initialize_service_epochs().await.unwrap(); + let registry = ServiceRegistryClient::from_shared(Arc::clone(&test.kv)); + registry + .register_chunkdb(1, &format!("127.0.0.1:{port}")) + .await + .unwrap(); + let client = ChunkdbClient::new(registry, Arc::new(ChunkdbRpcTransport::new())); + client.refresh_routes().await.unwrap(); + client +} + +fn request() -> AllocateChunkRequest { + AllocateChunkRequest { + chunk_id: Some(ChunkId { + high: 5 << 56, + low: 123, + }), + chunk_type: ChunkType::S3 as i32, + ..Default::default() + } +} + +#[tokio::test] +async fn known_quorum_unavailability_retries_allocation_with_the_same_chunk_id() { + let test = slot_groups::TestGroups::start().await; + let server = crowdb_rpc_ffi::RpcServer::new(None); + server.listen("127.0.0.1", 0).unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let observed = calls.clone(); + server.register_reply_for_tests(FBMsgType::EAllocateChunkRequest.0 as u16, move |request| { + let body = flatbuffers::root::(request.control()).unwrap(); + let id = body.chunk_id().unwrap(); + assert_eq!((id.high(), id.low()), (5 << 56, 123)); + let first = observed.fetch_add(1, Ordering::Relaxed) == 0; + let mut builder = flatbuffers::FlatBufferBuilder::new(); + let error = first.then(|| builder.create_string("linearizable read: leadership quorum unavailable")); + let chunk = (!first).then(|| { + FBChunk::create( + &mut builder, + &FBChunkArgs { + id: Some(id), + chunk_type: FBChunkType::S3, + ..Default::default() + }, + ) + }); + let response = FBAllocateChunkResponse::create( + &mut builder, + &FBAllocateChunkResponseArgs { + id: request.request_id, + rpc_create_nano: request.rpc_create_nano, + ret_code: if first { + FBChunkdbRetCode::Unavailable + } else { + FBChunkdbRetCode::Success + }, + error_msg: error, + chunk, + ..Default::default() + }, + ); + builder.finish(response, None); + (builder.finished_data().to_vec(), Vec::new()) + }); + server.start(); + let result = client(&test, server.port()) + .await + .allocate_chunk(request()) + .await + .unwrap(); + assert_eq!(result.chunk.unwrap().id, request().chunk_id); + assert_eq!(calls.load(Ordering::Relaxed), 2); + server.stop(); +} + +#[tokio::test] +async fn an_allocation_with_a_lost_reply_is_not_replayed() { + let test = slot_groups::TestGroups::start().await; + let server = crowdb_rpc_ffi::RpcServer::new(None); + server.listen("127.0.0.1", 0).unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let observed = calls.clone(); + server.register_handler(FBMsgType::EAllocateChunkRequest.0 as u16, move |_| { + observed.fetch_add(1, Ordering::Relaxed); + }); + server.start(); + let error = client(&test, server.port()) + .await + .allocate_chunk(request()) + .await + .unwrap_err(); + assert!(matches!(error, ChunkdbClientError::OutcomeUnknown(_)), "{error}"); + assert_eq!(calls.load(Ordering::Relaxed), 1); + server.stop(); +} diff --git a/app/crowdb-chunkdb/tests/allocator_owner_change_test.rs b/app/crowdb-chunkdb/tests/allocator_owner_change_test.rs new file mode 100644 index 000000000..e29eeab16 --- /dev/null +++ b/app/crowdb-chunkdb/tests/allocator_owner_change_test.rs @@ -0,0 +1,119 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[allow(dead_code)] +#[path = "common/slot_groups.rs"] +mod slot_groups; + +use crowdb_chunkdb::allocator::DiskdbClientPool; +use crowdb_diskdb_client::DiskdbClientError; +use crowdb_kv_client::ServiceRegistryClient; +use crowdb_protocol::{ + common::ChunkId, + diskdb_fb::{FBAllocateResponse, FBAllocateResponseArgs, FBDiskdbRetCode}, + fb::FBMsgType, +}; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +fn endpoint(code: FBDiskdbRetCode) -> (crowdb_rpc_ffi::RpcServer, Arc) { + let server = crowdb_rpc_ffi::RpcServer::new(None); + server.listen("127.0.0.1", 0).unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let observed = calls.clone(); + server.register_reply_for_tests(FBMsgType::EAllocateBlocksRequest.0 as u16, move |request| { + observed.fetch_add(1, Ordering::Relaxed); + let mut builder = flatbuffers::FlatBufferBuilder::new(); + let response = FBAllocateResponse::create( + &mut builder, + &FBAllocateResponseArgs { + id: request.request_id, + rpc_create_nano: request.rpc_create_nano, + ret_code: code, + ..Default::default() + }, + ); + builder.finish(response, None); + (builder.finished_data().to_vec(), Vec::new()) + }); + server.start(); + (server, calls) +} + +#[tokio::test] +async fn allocation_reroutes_after_an_explicit_old_owner_rejection() { + let test = slot_groups::TestGroups::start().await; + let registry = ServiceRegistryClient::from_shared(test.kv.clone()); + let (old, old_calls) = endpoint(FBDiskdbRetCode::NotOwner); + let (new, new_calls) = endpoint(FBDiskdbRetCode::Success); + registry + .register_diskdb(1, &format!("127.0.0.1:{}", old.port()), &[1], &[]) + .await + .unwrap(); + let pool = DiskdbClientPool::new(registry.clone()); + pool.refresh_endpoints().await.unwrap(); + registry + .register_diskdb(1, &format!("127.0.0.1:{}", new.port()), &[1], &[]) + .await + .unwrap(); + let result = pool + .allocate_blocks( + 1, + 1, + 1, + &ChunkId { + high: 5 << 56, + low: 1, + }, + ) + .await; + old.stop(); + new.stop(); + result.unwrap(); + assert_eq!(old_calls.load(Ordering::Relaxed), 1); + assert_eq!(new_calls.load(Ordering::Relaxed), 1); +} + +#[tokio::test] +async fn allocation_does_not_reroute_an_ambiguous_submitted_request() { + let test = slot_groups::TestGroups::start().await; + let registry = ServiceRegistryClient::from_shared(test.kv.clone()); + let old = crowdb_rpc_ffi::RpcServer::new(None); + old.listen("127.0.0.1", 0).unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let observed = calls.clone(); + old.register_handler(FBMsgType::EAllocateBlocksRequest.0 as u16, move |_| { + observed.fetch_add(1, Ordering::Relaxed); + }); + old.start(); + let (new, new_calls) = endpoint(FBDiskdbRetCode::Success); + registry + .register_diskdb(1, &format!("127.0.0.1:{}", old.port()), &[1], &[]) + .await + .unwrap(); + let pool = DiskdbClientPool::new(registry.clone()); + pool.refresh_endpoints().await.unwrap(); + registry + .register_diskdb(1, &format!("127.0.0.1:{}", new.port()), &[1], &[]) + .await + .unwrap(); + let error = pool + .allocate_blocks( + 1, + 1, + 1, + &ChunkId { + high: 5 << 56, + low: 2, + }, + ) + .await + .unwrap_err(); + old.stop(); + new.stop(); + assert!(matches!(error, DiskdbClientError::Unreachable(_)), "{error}"); + assert_eq!(calls.load(Ordering::Relaxed), 1); + assert_eq!(new_calls.load(Ordering::Relaxed), 0); +} diff --git a/app/crowdb-chunkdb/tests/common/http_readiness.rs b/app/crowdb-chunkdb/tests/common/http_readiness.rs new file mode 100644 index 000000000..f398ac4ca --- /dev/null +++ b/app/crowdb-chunkdb/tests/common/http_readiness.rs @@ -0,0 +1,96 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use axum::{body::Body, http::Request}; +use crowdb_chunkdb::{range_guard::RangeGuard, readiness::RangeReadiness}; +use crowdb_protocol::chunk_slot::{ChunkSlotBootstrap, ChunkStorageGroup}; +use std::{ + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::{sync::watch, task::JoinHandle}; +use tower::ServiceExt; + +pub struct TestReadiness { + pub guard: Arc, + pub readiness: Arc, + pub mode: Arc, + pub calls: Arc, + pub stop: watch::Sender, + pub worker: JoinHandle<()>, +} + +impl TestReadiness { + pub fn new(initialized: bool, instance: u64) -> Self { + let guard = Arc::new(RangeGuard::new()); + if initialized { + guard.install(&layout().service_map().unwrap(), instance).unwrap(); + } + let readiness = Arc::new(RangeReadiness::new(Arc::clone(&guard))); + let mode = Arc::new(AtomicUsize::new(0)); + let calls = Arc::new(AtomicUsize::new(0)); + let (stop, receiver) = watch::channel(false); + let worker = Arc::clone(&readiness).start_source_for_tests(receiver, { + let mode = Arc::clone(&mode); + let calls = Arc::clone(&calls); + move || { + let mode = mode.load(Ordering::Relaxed); + calls.fetch_add(1, Ordering::Relaxed); + async move { + match mode { + 1 => tokio::time::sleep(Duration::from_millis(1500)).await, + 2 => std::future::pending::<()>().await, + 3 => return Err("authority unavailable".into()), + _ => {} + } + Ok(()) + } + } + }); + Self { + guard, + readiness, + mode, + calls, + stop, + worker, + } + } + + pub async fn read(&self) -> u16 { + read(Arc::clone(&self.readiness)).await + } + + pub async fn settle() { + for _ in 0..16 { + tokio::task::yield_now().await; + } + } +} + +pub fn layout() -> ChunkSlotBootstrap { + ChunkSlotBootstrap { + service_instances: vec![1], + storage_groups: vec![ChunkStorageGroup { + store_id: 0, + group_id: 1, + }], + } +} + +pub async fn read(readiness: Arc) -> u16 { + tokio::time::timeout( + Duration::from_secs(1), + readiness + .router() + .oneshot(Request::builder().uri("/ready").body(Body::empty()).unwrap()), + ) + .await + .expect("HTTP readiness waited for authority I/O") + .unwrap() + .status() + .as_u16() +} diff --git a/app/crowdb-chunkdb/tests/common/readiness_failures.rs b/app/crowdb-chunkdb/tests/common/readiness_failures.rs new file mode 100644 index 000000000..c50dc1fd9 --- /dev/null +++ b/app/crowdb-chunkdb/tests/common/readiness_failures.rs @@ -0,0 +1,47 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_chunkdb::range_guard::RangeGuard; +use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbClientError}; +use crowdb_kv_client::CrowdbKvClient; +use crowdb_protocol::chunk_id::{ChunkIdParts, CHUNK_TYPE_S3}; +use crowdb_rpc_ffi::RpcServer; +use std::time::Duration; + +pub async fn assert_storage_and_owner_failures( + client: &ChunkdbClient, + data: &CrowdbKvClient, + control: &CrowdbKvClient, + guard: &RangeGuard, + server: &RpcServer, +) { + // Corruption is an error, not an authoritative absent key. + let mut key = b"/chunk/".to_vec(); + key.extend_from_slice( + &ChunkIdParts { + high: u64::from(CHUNK_TYPE_S3) << 56, + low: 0, + } + .to_bytes(), + ); + data.put(0, 1, &key, b"corrupt chunk record", None).await.unwrap(); + assert!(matches!( + Box::pin(client.observe_readiness()).await, + Err(ChunkdbClientError::Internal(_)) + )); + // A stale cached owner rejects the probe and requests authoritative refresh. + guard.replace_for_tests(&[]); + let _ = control.drain_window(); + let rejected = + tokio::time::timeout(Duration::from_millis(750), Box::pin(client.observe_readiness())).await; + let rescanned = control.drain_window().scan.len(); + server.stop(); + assert!( + !matches!(rejected, Ok(Ok(()))), + "a rejected owner cannot be Ready" + ); + assert!( + rescanned > 0, + "ownership rejection must still refresh the registry" + ); +} diff --git a/app/crowdb-chunkdb/tests/common/readiness_registry.rs b/app/crowdb-chunkdb/tests/common/readiness_registry.rs new file mode 100644 index 000000000..adf4896fc --- /dev/null +++ b/app/crowdb-chunkdb/tests/common/readiness_registry.rs @@ -0,0 +1,35 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_chunkdb_client::ChunkdbClient; +use crowdb_kv_client::CrowdbKvClient; +use crowdb_rpc_ffi::RpcServer; +use std::time::Duration; + +pub async fn assert_warm_registry_independence( + client: &ChunkdbClient, + control: &CrowdbKvClient, + data: &CrowdbKvClient, + server: &RpcServer, + endpoint: String, +) { + // Keep the real owner/storage healthy while the warmed client's Group0 + // route fails after ordinary registry refresh becomes due. + tokio::time::sleep(Duration::from_millis(5100)).await; + control.seed_leader(0, 0, "127.0.0.1:0".into()); + let _ = control.drain_window(); + let _ = data.drain_window(); + let observed = + tokio::time::timeout(Duration::from_millis(750), Box::pin(client.observe_readiness())).await; + control.seed_leader(0, 0, endpoint); + if !matches!(observed, Ok(Ok(()))) { + server.stop(); + } + assert!( + matches!(observed, Ok(Ok(()))), + "a validated route and current point reads must not wait for a registry scan: {observed:?}" + ); + let work = control.drain_window(); + assert_eq!(work.get.len() + work.scan.len(), 0); + assert_eq!(data.drain_window().get.len(), 2); +} diff --git a/app/crowdb-chunkdb/tests/common/repair_targets.rs b/app/crowdb-chunkdb/tests/common/repair_targets.rs new file mode 100644 index 000000000..cf9e15e4b --- /dev/null +++ b/app/crowdb-chunkdb/tests/common/repair_targets.rs @@ -0,0 +1,270 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use crate::common::cluster::{ + seed_hardware, ChunkdbHarness, DiskdbServer, KvCluster, DATA_GROUP_ID, STORE_ID, +}; +use crate::common::diskio_stub::TestDiskio; +use crowdb_chunkdb::conversion::io::ConversionDiskIo; +use crowdb_chunkdb::metrics::ChunkdbMetrics; +use crowdb_chunkdb::repair::{decode_payload, encode_payload, RepairCoordinator, RepairStripTaskHandler}; +use crowdb_chunkdb::routing::{default_binding_table, BindingCache}; +use crowdb_chunkdb::task::{TaskExecutor, TaskManager, TaskStore}; +use crowdb_chunkdb::topology::build_snapshot; +use crowdb_common::metrics::MetricsRegistry; +use crowdb_kv_client::{GetOutcome, ReadMode}; +use crowdb_protocol::chunk_task::{ChunkTaskValue, RepairTargetCheckpoint, RepairTargetPhase}; +use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkType, Strip, StripType}; +use crowdb_protocol::common::{ChunkId, InstanceValue}; +use crowdb_protocol::diskdb::rpc::Segment; +use crowdb_protocol::key::InstanceKey; + +pub struct TestRepairTargets { + pub cluster: KvCluster, + pub harness: ChunkdbHarness, + pub tasks: Arc, + pub manager: Arc, + pub io: Arc, + pub chunk: Chunk, + pub failed: Segment, + pub survivor: Segment, + pub target: Segment, + pub task: ChunkTaskValue, + disks: Vec<(u64, Option)>, + _diskdb: DiskdbServer, +} + +impl TestRepairTargets { + pub async fn start() -> Self { + let cluster = KvCluster::start_pinned().await; + let hardware = cluster.make_hardware_client(); + seed_hardware(&hardware).await; + let diskdb = DiskdbServer::start(&cluster).await; + let registry = cluster.make_service_registry_client(); + let mut disks = Vec::new(); + for group in hardware.list_disk_groups().await.unwrap() { + let disk = + TestDiskio::start(®istry, group.dg_id, group.rack_id, group.node_id, group.dg_id).await; + disks.push((group.dg_id, Some(disk))); + } + let harness = ChunkdbHarness::start(&cluster).await; + let chunk = harness + .handler + .allocate_chunk(None, 1, 1, StripType::Mirror, 0, 0, 2, ChunkType::S3, 0, 0) + .await + .unwrap(); + let Strip::MirrorStrip(mirror) = chunk.strips[0].strip.as_ref().unwrap() else { + panic!("expected mirrored chunk"); + }; + let (failed, survivor) = (mirror.segments[0], mirror.segments[1]); + let id = chunk.id.unwrap(); + let target = harness + .handler + .allocate_repair_segment( + &id, + &failed, + &[survivor], + &[failed.disk_id.unwrap(), survivor.disk_id.unwrap()], + false, + ) + .await + .unwrap(); + let io = Arc::new(ConversionDiskIo::connect(®istry, &hardware).await.unwrap()); + let unit = u64::from(chunk.strips[0].unit_kb) * 1024; + let data = bytes::Bytes::from(vec![42; usize::try_from(unit).unwrap()]); + io.write_segment(&survivor, unit, data).await.unwrap(); + io.fsync_segment(&survivor).await.unwrap(); + let marked = mark_failed(&harness, &chunk, failed).await; + let bindings = BindingCache::new(); + bindings + .replace(default_binding_table(STORE_ID, DATA_GROUP_ID)) + .unwrap(); + let tasks = Arc::new(TaskStore::new(cluster.make_crowdb_client(), bindings)); + let task = persist_allocated_target(&harness, &tasks, &marked, target).await; + let manager = Arc::new(TaskManager::new(Arc::clone(&tasks), 8001, 30_000)); + Self { + cluster, + harness, + tasks, + manager, + io, + chunk: marked, + failed, + survivor, + target, + task, + disks, + _diskdb: diskdb, + } + } + + pub async fn stop_target(&mut self, expire_registration: bool) { + let group = self + .harness + .topology + .snapshot() + .disk_location(self.target.disk_id.unwrap()) + .unwrap() + .disk_group_id; + let key = InstanceKey { + service: "diskio".into(), + instance_id: group, + } + .to_path(); + if expire_registration { + let kv = self.cluster.make_crowdb_client(); + let GetOutcome::Found { value, .. } = kv + .get(0, 0, key.as_bytes(), ReadMode::Linearizable, None) + .await + .unwrap() + else { + panic!("missing DiskIO registration"); + }; + let mut instance: InstanceValue = serde_json::from_slice(&value).unwrap(); + instance.last_heartbeat_ms = 0; + kv.put( + 0, + 0, + key.as_bytes(), + &serde_json::to_vec(&instance).unwrap(), + None, + ) + .await + .unwrap(); + } + self.disks + .iter_mut() + .find(|(id, _)| *id == group) + .unwrap() + .1 + .take(); + let snapshot = build_snapshot(&self.cluster.make_hardware_client()) + .await + .unwrap(); + assert_eq!( + snapshot.healthy_disk_groups().iter().any(|g| g.dg_id == group), + !expire_registration + ); + self.harness.topology.replace(snapshot); + } + + pub async fn execute(&self) -> ChunkTaskValue { + let ready = self.tasks.scan_ready(now_ms(), 16).await.unwrap(); + let index = ready + .iter() + .find(|index| index.task_id == self.task.task_id) + .unwrap(); + let claim = self.manager.claim(index, now_ms()).await.unwrap().unwrap(); + let metrics = ChunkdbMetrics::register(&mut MetricsRegistry::new()).repair; + let worker = RepairStripTaskHandler::new( + Arc::clone(&self.harness.handler), + Arc::clone(&self.manager), + Arc::clone(&self.io), + 16 * 1024 * 1024, + 1, + false, + metrics, + ); + TaskExecutor::new(Arc::clone(&self.manager), 1, vec![Arc::new(worker)]) + .unwrap() + .execute(claim) + .await + .unwrap(); + self.tasks + .get(&self.task.partition_id, self.task.kind, &self.task.task_id) + .await + .unwrap() + .unwrap() + } + + pub async fn publish_before_checkpoint(&self) -> Chunk { + let unit = u64::from(self.chunk.strips[0].unit_kb) * 1024; + let bytes = self.io.read_segment(&self.survivor, unit).await.unwrap(); + self.io.write_segment(&self.target, unit, bytes).await.unwrap(); + self.io.fsync_segment(&self.target).await.unwrap(); + let mut replacement = self.chunk.strips[0].clone(); + let Strip::MirrorStrip(mirror) = replacement.strip.as_mut().unwrap() else { + unreachable!() + }; + *mirror + .segments + .iter_mut() + .find(|segment| **segment == self.failed) + .unwrap() = self.target; + replacement.unavailable_segments.clear(); + self.harness + .handler + .publish_tentative_chunk_strip_range( + &self.chunk.id.unwrap(), + self.chunk.modify_ts, + 0, + &self.chunk.strips, + &[replacement], + self.task.operation_id, + ) + .await + .unwrap() + } +} + +async fn mark_failed(harness: &ChunkdbHarness, chunk: &Chunk, failed: Segment) -> Chunk { + let mut marked = chunk.strips[0].clone(); + marked.unavailable_segments.push(failed); + harness + .handler + .replace_chunk_strip_range( + &chunk.id.unwrap(), + chunk.modify_ts, + 0, + &chunk.strips, + &[marked], + ChunkId { high: 111, low: 1 }, + ) + .await + .unwrap() +} + +async fn persist_allocated_target( + harness: &ChunkdbHarness, + tasks: &Arc, + chunk: &Chunk, + target: Segment, +) -> ChunkTaskValue { + RepairCoordinator::new(Arc::clone(&harness.handler), Arc::clone(tasks)) + .admit_chunk(chunk, now_ms()) + .await + .unwrap(); + let ready = tasks.scan_ready(now_ms(), 16).await.unwrap(); + let index = ready + .iter() + .find(|index| index.kind == crowdb_protocol::chunk_task::TASK_KIND_REPAIR_STRIP) + .unwrap(); + let task = tasks + .get(&chunk.id.unwrap(), index.kind, &index.task_id) + .await + .unwrap() + .unwrap(); + let mut payload = decode_payload(&task.payload).unwrap(); + payload.targets.push(RepairTargetCheckpoint { + source: payload.failed_segments[0], + destination: target, + phase: RepairTargetPhase::Allocated, + }); + let mut checkpoint = task.clone(); + checkpoint.revision += 1; + checkpoint.payload = encode_payload(&payload).unwrap(); + tasks.write_transition(Some(&task), &checkpoint).await.unwrap(); + checkpoint +} + +fn now_ms() -> u64 { + u64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(), + ) + .unwrap() +} diff --git a/app/crowdb-chunkdb/tests/common/topology_availability.rs b/app/crowdb-chunkdb/tests/common/topology_availability.rs new file mode 100644 index 000000000..6efba4a20 --- /dev/null +++ b/app/crowdb-chunkdb/tests/common/topology_availability.rs @@ -0,0 +1,23 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_kv_client::ServiceRegistryClient; +use crowdb_protocol::{common::DiskioHardwareScope, sysdata::DiskGroupEntry}; + +pub async fn heartbeat(registry: &ServiceRegistryClient, group: &DiskGroupEntry) { + registry + .heartbeat_diskio_scoped( + group.dg_id, + "127.0.0.1:13000", + &DiskioHardwareScope { + version: 1, + rack_id: group.rack_id, + node_id: group.node_id, + disk_group_id: group.dg_id, + }, + &[group.dg_id], + &[], + ) + .await + .unwrap(); +} diff --git a/app/crowdb-chunkdb/tests/http_readiness_test.rs b/app/crowdb-chunkdb/tests/http_readiness_test.rs new file mode 100644 index 000000000..ebaa33153 --- /dev/null +++ b/app/crowdb-chunkdb/tests/http_readiness_test.rs @@ -0,0 +1,118 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/http_readiness.rs"] +mod http_readiness; +#[path = "common/slot_groups.rs"] +#[allow(dead_code)] +mod slot_groups; + +use crowdb_chunkdb::{range_guard::RangeGuard, readiness::RangeReadiness}; +use crowdb_kv_client::ChunkSlotMapClient; +use http_readiness::TestReadiness; +use std::{ + sync::{atomic::Ordering, Arc}, + time::Duration, +}; + +#[tokio::test] +async fn http_readers_share_real_validated_ownership_without_group_zero_work() { + let groups = slot_groups::TestGroups::start().await; + ChunkSlotMapClient::new(Arc::clone(&groups.kv)) + .initialize_service(&http_readiness::layout().service_map().unwrap()) + .await + .unwrap(); + let guard = Arc::new(RangeGuard::new()); + let readiness = Arc::new(RangeReadiness::new(guard)); + assert_eq!(http_readiness::read(Arc::clone(&readiness)).await, 503); + let (stop, receiver) = tokio::sync::watch::channel(false); + let worker = Arc::clone(&readiness).start(Arc::clone(&groups.kv), 1, receiver); + tokio::time::timeout(Duration::from_secs(3), async { + while http_readiness::read(Arc::clone(&readiness)).await != 200 { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("background ownership sample did not finish"); + stop.send(true).unwrap(); + worker.await.unwrap(); + let _ = groups.kv.drain_window(); + let responses = + futures::future::join_all((0..32).map(|_| http_readiness::read(Arc::clone(&readiness)))).await; + assert!(responses.iter().all(|status| *status == 200)); + let work = groups.kv.drain_window(); + assert_eq!( + work.get.len() + work.scan.len() + work.put.len() + work.batch_write.len(), + 0 + ); + tokio::time::sleep(Duration::from_millis(3050)).await; + assert_eq!( + http_readiness::read(readiness).await, + 503, + "readers cannot renew evidence" + ); +} + +#[tokio::test(start_paused = true)] +async fn slow_ownership_refresh_is_independent_of_http_readers() { + let test = TestReadiness::new(true, 1); + test.mode.store(1, Ordering::Relaxed); + TestReadiness::settle().await; + assert_eq!(test.calls.load(Ordering::Relaxed), 1); + for _ in 0..32 { + assert_eq!(test.read().await, 503); + } + assert_eq!(test.calls.load(Ordering::Relaxed), 1); + tokio::time::advance(Duration::from_millis(1500)).await; + TestReadiness::settle().await; + assert_eq!( + test.read().await, + 200, + "a completed valid sample is readable within one second" + ); + test.stop.send(true).unwrap(); + test.worker.await.unwrap(); +} + +#[tokio::test(start_paused = true)] +async fn stalled_authority_expires_without_renewal_and_recovers_without_reader_traffic() { + let test = TestReadiness::new(true, 1); + TestReadiness::settle().await; + assert_eq!(test.read().await, 200); + test.mode.store(2, Ordering::Relaxed); + tokio::time::advance(Duration::from_secs(1)).await; + TestReadiness::settle().await; + assert_eq!(test.read().await, 200); + tokio::time::advance(Duration::from_millis(2050)).await; + TestReadiness::settle().await; + assert_eq!(test.read().await, 503); + tokio::time::advance(Duration::from_secs(1)).await; + TestReadiness::settle().await; + assert_eq!( + test.read().await, + 503, + "deadline errors cannot publish healthy evidence" + ); + test.mode.store(0, Ordering::Relaxed); + tokio::time::advance(Duration::from_secs(3)).await; + TestReadiness::settle().await; + assert_eq!(test.read().await, 200); + test.stop.send(true).unwrap(); + test.worker.await.unwrap(); +} + +#[tokio::test(start_paused = true)] +async fn authority_errors_and_uninitialized_maps_are_unready_but_idle_initialized_owners_are_ready() { + for (initialized, instance, expected) in [(false, 1, 503), (true, 2, 200)] { + let test = TestReadiness::new(initialized, instance); + TestReadiness::settle().await; + assert_eq!(test.read().await, expected); + assert!(test.guard.is_empty()); + test.mode.store(3, Ordering::Relaxed); + tokio::time::advance(Duration::from_secs(1)).await; + TestReadiness::settle().await; + assert_eq!(test.read().await, 503); + test.stop.send(true).unwrap(); + test.worker.await.unwrap(); + } +} diff --git a/app/crowdb-chunkdb/tests/maintenance_domain_scan_test.rs b/app/crowdb-chunkdb/tests/maintenance_domain_scan_test.rs new file mode 100644 index 000000000..0bd200843 --- /dev/null +++ b/app/crowdb-chunkdb/tests/maintenance_domain_scan_test.rs @@ -0,0 +1,128 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_chunkdb::range_guard::RangeGuard; +use crowdb_chunkdb::routing::{BindingCache, BindingTable}; +use crowdb_chunkdb::storage::ChunkStore; +use crowdb_protocol::chunk_domain::ChunkDomain; +use crowdb_protocol::chunk_slot::{ChunkSlotBootstrap, ChunkStorageGroup}; +use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkState, StripReservationGroup}; +use crowdb_protocol::common::ChunkId; +use std::sync::Arc; + +#[path = "common/slot_groups.rs"] +#[allow(dead_code)] +mod slot_groups; +use slot_groups::TestGroups; + +async fn seed(writer: &ChunkStore, purpose: u8, low: u64) -> ChunkId { + let id = ChunkId { + high: u64::from(purpose) << 56, + low, + }; + writer + .put_chunk(&Chunk { + id: Some(id), + modify_ts: 1, + state: ChunkState::Active as i32, + chunk_type: i32::from(purpose), + ..Default::default() + }) + .await + .unwrap(); + writer + .put_reservation_group(&StripReservationGroup { + chunk_id: Some(id), + group_id: Some(id), + ..Default::default() + }) + .await + .unwrap(); + id +} + +async fn assert_domain_scan(domain: ChunkDomain, foreign: u8, purposes: &[u8]) { + let test = TestGroups::start().await; + let layout = ChunkSlotBootstrap { + service_instances: vec![1], + storage_groups: vec![ChunkStorageGroup { + store_id: 0, + group_id: 1, + }], + }; + let bindings = BindingCache::new(); + bindings + .replace(BindingTable::new(layout.storage_map().unwrap())) + .unwrap(); + let guard = Arc::new(RangeGuard::new()); + guard.install(&layout.service_map().unwrap(), 1).unwrap(); + let writer = ChunkStore::new(Arc::clone(&test.kv), bindings.clone()); + let reader = ChunkStore::new(Arc::clone(&test.kv), bindings).with_scope(guard, domain); + for low in 1..=600 { + seed(&writer, foreign, low).await; + } + let mut expected = Vec::new(); + for purpose in purposes { + expected.push(seed(&writer, *purpose, 0).await); + } + let _ = test.kv.drain_window(); + let chunks = reader.list_chunks(None, 100).await.unwrap(); + assert_eq!( + chunks.iter().map(|chunk| chunk.id.unwrap()).collect::>(), + expected + ); + assert_eq!( + test.kv.drain_window().scan.len(), + 1, + "a scoped scan must not transfer pages belonging only to another maintenance domain" + ); + let reservations = reader.scan_reservation_groups(100).await.unwrap(); + assert_eq!( + reservations + .iter() + .map(|group| group.chunk_id.unwrap()) + .collect::>(), + expected + ); + assert_eq!(test.kv.drain_window().scan.len(), 1); + assert_eq!( + writer.list_chunks(None, 1000).await.unwrap().len(), + 600 + expected.len() + ); + for (index, id) in expected.iter().enumerate() { + let page = reader.list_chunks(Some(id), 1).await.unwrap(); + assert_eq!( + page.first().and_then(|chunk| chunk.id), + expected.get(index + 1).copied() + ); + let page = reader + .scan_reservation_groups_after(1, Some((id, id))) + .await + .unwrap(); + assert_eq!( + page.first().and_then(|group| group.chunk_id), + expected.get(index + 1).copied() + ); + } + let after_domain = ChunkId { + high: 8 << 56, + low: 0, + }; + let _ = test.kv.drain_window(); + assert!(reader + .list_chunks(Some(&after_domain), 1) + .await + .unwrap() + .is_empty()); + assert!(test.kv.drain_window().scan.is_empty()); +} + +#[tokio::test] +async fn user_data_scans_exclude_system_pages_before_transfer() { + Box::pin(assert_domain_scan(ChunkDomain::UserData, 1, &[4, 5, 6, 7])).await; +} + +#[tokio::test] +async fn system_scans_exclude_user_data_pages_before_transfer() { + Box::pin(assert_domain_scan(ChunkDomain::System, 5, &[1, 2, 3])).await; +} diff --git a/app/crowdb-chunkdb/tests/readiness_probe_test.rs b/app/crowdb-chunkdb/tests/readiness_probe_test.rs index eb9430b19..cc34224e1 100644 --- a/app/crowdb-chunkdb/tests/readiness_probe_test.rs +++ b/app/crowdb-chunkdb/tests/readiness_probe_test.rs @@ -8,17 +8,21 @@ use crowdb_chunkdb::{ routing::{BindingCache, BindingTable}, storage::ChunkStore, }; -use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbClientError, ChunkdbRpcTransport}; +use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbRpcTransport}; use crowdb_kv::cluster::kv_server::KvServer; use crowdb_kv_client::{ChunkSlotMapClient, ClientConfig, CrowdbKvClient, ServiceRegistryClient}; use crowdb_protocol::{ chunk_domain::ChunkDomain, - chunk_id::{ChunkIdParts, CHUNK_TYPE_S3}, + chunk_id::CHUNK_TYPE_S3, chunk_slot::{ChunkSlotBootstrap, ChunkStorageGroup}, chunkdb::rpc::{Chunk, ChunkState}, common::ChunkId, }; +#[path = "common/readiness_failures.rs"] +mod readiness_failures; +#[path = "common/readiness_registry.rs"] +mod readiness_registry; #[path = "common/runtime_context.rs"] #[allow(dead_code)] mod runtime_context; @@ -46,7 +50,8 @@ async fn readiness_reads_both_domains_without_scanning_or_mutating_the_catalog() let guard = Arc::new(RangeGuard::new()); guard.install(&layout.service_map().unwrap(), 1).unwrap(); let store = ChunkStore::new(Arc::clone(&test.kv), bindings.clone()); - let (context, stop) = runtime_context::runtime_context(Arc::clone(&test.kv), bindings, guard); + let (context, stop) = + runtime_context::runtime_context(Arc::clone(&test.kv), bindings, Arc::clone(&guard)); let system = context.clone().start(ChunkDomain::System).await.unwrap(); let user = context.start(ChunkDomain::UserData).await.unwrap(); let service = Arc::new(user.rpc_service.with_system_runtime(Arc::new(system.rpc_service))); @@ -69,6 +74,7 @@ async fn readiness_reads_both_domains_without_scanning_or_mutating_the_catalog() .await .unwrap(); let client = ChunkdbClient::new(registry, Arc::new(ChunkdbRpcTransport::new())); + Box::pin(client.observe_readiness()).await.unwrap(); let reference = crowdb_kv_client::RangeBindingClient::from_shared(Arc::clone(&control)); let _ = control.drain_window(); reference.refresh().await.unwrap(); @@ -89,25 +95,16 @@ async fn readiness_reads_both_domains_without_scanning_or_mutating_the_catalog() assert_eq!(work.scan.len(), 0, "readiness must not enumerate catalog records"); assert_eq!(work.put.len() + work.batch_write.len() + work.delete.len(), 0); } - // Storage corruption must remain an unhealthy result, rather than being - // confused with an authoritative absent key in an empty catalog. - let mut key = b"/chunk/".to_vec(); - key.extend_from_slice( - &ChunkIdParts { - high: u64::from(CHUNK_TYPE_S3) << 56, - low: 0, - } - .to_bytes(), - ); - test.kv - .put(0, 1, &key, b"corrupt chunk record", None) - .await - .unwrap(); - assert!(matches!( - Box::pin(client.observe_readiness()).await, - Err(ChunkdbClientError::Internal(_)) - )); - server.stop(); + readiness_registry::assert_warm_registry_independence( + &client, + &control, + &test.kv, + &server, + test.server.listen_addr().unwrap().to_string(), + ) + .await; + + readiness_failures::assert_storage_and_owner_failures(&client, &test.kv, &control, &guard, &server).await; } async fn seed_catalog(store: &ChunkStore, count: u64) { diff --git a/app/crowdb-chunkdb/tests/repair_target_recovery_test.rs b/app/crowdb-chunkdb/tests/repair_target_recovery_test.rs new file mode 100644 index 000000000..f559c1926 --- /dev/null +++ b/app/crowdb-chunkdb/tests/repair_target_recovery_test.rs @@ -0,0 +1,110 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +mod common; +#[path = "common/repair_targets.rs"] +mod repair_targets; + +use crowdb_chunkdb::repair::decode_payload; +use crowdb_chunkdb::task::SegmentOwnerResolver; +use crowdb_kv_client::{GetOutcome, ReadMode}; +use crowdb_protocol::chunk_task::{ChunkTaskState, RepairTargetPhase}; +use crowdb_protocol::chunkdb::rpc::{SegmentOwnerDisposition, Strip}; +use crowdb_protocol::key::{BinaryKey, BusyBlockKey}; +use repair_targets::TestRepairTargets; +use std::sync::Arc; + +#[tokio::test] +async fn expired_unpublished_repair_target_is_replaced_without_freeing_its_allocation() { + let mut test = TestRepairTargets::start().await; + test.stop_target(true).await; + let task = test.execute().await; + assert_eq!(task.state, ChunkTaskState::Completed, "{}", task.last_error); + let chunk = test + .harness + .handler + .query_chunk(&test.chunk.id.unwrap()) + .await + .unwrap(); + let Strip::MirrorStrip(mirror) = chunk.strips[0].strip.as_ref().unwrap() else { + unreachable!() + }; + assert!(!mirror.segments.contains(&test.failed)); + assert!(!mirror.segments.contains(&test.target)); + let replacement = mirror + .segments + .iter() + .find(|segment| **segment != test.survivor) + .unwrap(); + let unit = u64::from(chunk.strips[0].unit_kb) * 1024; + assert_eq!( + test.io.read_segment(replacement, unit).await.unwrap().as_ref(), + vec![42; usize::try_from(unit).unwrap()] + ); + let owner = SegmentOwnerResolver::new(Arc::clone(&test.harness.handler), Arc::clone(&test.tasks)); + assert_eq!( + owner + .resolve(&test.chunk.id.unwrap(), &test.target) + .await + .unwrap(), + SegmentOwnerDisposition::Absent + ); + let key = BusyBlockKey { + disk_id: test.target.disk_id.unwrap(), + zone_index: test.target.zone_index, + unit_offset: test.target.unit_offset, + } + .to_bytes(); + assert!( + matches!( + test.cluster + .make_crowdb_client() + .get(0, 1, &key, ReadMode::Linearizable, None) + .await + .unwrap(), + GetOutcome::Found { .. } + ), + "unpublished allocation must remain for the DiskDB owner scanner grace period" + ); +} + +#[tokio::test] +async fn live_unpublished_repair_target_is_retained_after_a_transport_failure() { + let mut test = TestRepairTargets::start().await; + test.stop_target(false).await; + let task = test.execute().await; + assert_eq!(task.state, ChunkTaskState::RetryWait); + let payload = decode_payload(&task.payload).unwrap(); + assert_eq!(payload.targets.len(), 1); + assert_eq!(payload.targets[0].destination, test.target); + assert_eq!(payload.targets[0].phase, RepairTargetPhase::Allocated); + let owner = SegmentOwnerResolver::new(Arc::clone(&test.harness.handler), Arc::clone(&test.tasks)); + assert_eq!( + owner + .resolve(&test.chunk.id.unwrap(), &test.target) + .await + .unwrap(), + SegmentOwnerDisposition::TaskPending + ); +} + +#[tokio::test] +async fn published_repair_target_survives_expiry_before_its_task_checkpoint() { + let mut test = TestRepairTargets::start().await; + let published = test.publish_before_checkpoint().await; + test.stop_target(true).await; + let task = test.execute().await; + assert_eq!(task.state, ChunkTaskState::Completed, "{}", task.last_error); + let payload = decode_payload(&task.payload).unwrap(); + assert_eq!(payload.targets.len(), 1); + assert_eq!(payload.targets[0].destination, test.target); + assert_eq!(payload.targets[0].phase, RepairTargetPhase::Confirmed); + assert_eq!( + test.harness + .handler + .query_chunk(&test.chunk.id.unwrap()) + .await + .unwrap(), + published + ); +} diff --git a/app/crowdb-chunkdb/tests/topology_availability_test.rs b/app/crowdb-chunkdb/tests/topology_availability_test.rs new file mode 100644 index 000000000..8bf884342 --- /dev/null +++ b/app/crowdb-chunkdb/tests/topology_availability_test.rs @@ -0,0 +1,98 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/topology_availability.rs"] +mod availability; +mod common; + +use crowdb_chunkdb::{ + selector::{MirrorPlacement, PlacementConstraints}, + topology::build_snapshot, +}; +use crowdb_kv_client::ServiceRegistryClient; +use crowdb_protocol::{ + common::{HwStatus, InstanceValue}, + key::InstanceKey, +}; + +#[tokio::test] +async fn expired_physical_io_is_excluded_without_changing_hardware_and_returns_after_heartbeat() { + let cluster = common::cluster::KvCluster::start_pinned().await; + let hardware = cluster.make_hardware_client(); + common::cluster::seed_hardware(&hardware).await; + let registry = ServiceRegistryClient::from_shared(cluster.make_crowdb_client()); + let groups = hardware.list_disk_groups().await.unwrap(); + assert!(groups.len() >= 3); + for group in &groups { + availability::heartbeat(®istry, group).await; + } + let before = build_snapshot(&hardware).await.unwrap(); + assert_eq!(before.healthy_disk_groups().len(), groups.len()); + let expired = u64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(), + ) + .unwrap() + + crowdb_kv_client::SERVICE_INSTANCE_TTL_MS + + 1; + assert!(before.healthy_disk_groups_at_for_tests(expired).is_empty()); + let stopped = &groups[0]; + let key = InstanceKey { + service: "diskio".into(), + instance_id: stopped.dg_id, + } + .to_path(); + let kv = cluster.make_crowdb_client(); + let value = kv + .get( + 0, + 0, + key.as_bytes(), + crowdb_kv_client::ReadMode::Linearizable, + None, + ) + .await + .unwrap(); + let crowdb_kv_client::GetOutcome::Found { value, .. } = value else { + panic!("missing IO registration") + }; + let mut observation: InstanceValue = serde_json::from_slice(&value).unwrap(); + observation.last_heartbeat_ms = 0; + kv.put( + 0, + 0, + key.as_bytes(), + &serde_json::to_vec(&observation).unwrap(), + None, + ) + .await + .unwrap(); + let unavailable = build_snapshot(&hardware).await.unwrap(); + assert!(unavailable + .healthy_disk_groups() + .iter() + .all(|group| group.dg_id != stopped.dg_id)); + let plan = MirrorPlacement::select(&unavailable, 2, &PlacementConstraints::default()).unwrap(); + assert!(plan + .entries + .iter() + .all(|entry| entry.disk_group_id != stopped.dg_id)); + assert_eq!(before.healthy_disk_groups().len(), groups.len()); + assert!(hardware + .list_disk_groups() + .await + .unwrap() + .iter() + .all(|group| group.value.status == HwStatus::Up as i32)); + availability::heartbeat(®istry, stopped).await; + assert_eq!( + build_snapshot(&hardware) + .await + .unwrap() + .healthy_disk_groups() + .len(), + groups.len() + ); +} diff --git a/app/crowdb-cli/src/commands/cluster/local.rs b/app/crowdb-cli/src/commands/cluster/local.rs index b16b55417..b8e87764c 100644 --- a/app/crowdb-cli/src/commands/cluster/local.rs +++ b/app/crowdb-cli/src/commands/cluster/local.rs @@ -216,7 +216,7 @@ async fn stop_inner(cli: &Cli) -> Result<()> { .ok_or_else(|| invalid("selected cluster has no local runtime ownership receipt"))?; binding(&record)?; Docker.check(&record).await?; - for item in Docker.objects("container").await? { + for item in Docker.objects(&record, "container").await? { if owned(&item, &record, "container") && item["State"]["Running"] == true { let identity = item["Id"] .as_str() diff --git a/app/crowdb-cli/src/commands/cluster/local/docker.rs b/app/crowdb-cli/src/commands/cluster/local/docker.rs index 40964c805..b4bc25b47 100644 --- a/app/crowdb-cli/src/commands/cluster/local/docker.rs +++ b/app/crowdb-cli/src/commands/cluster/local/docker.rs @@ -70,21 +70,30 @@ impl Docker { } Ok(()) } - pub async fn objects(&self, kind: &str) -> Result> { + pub async fn objects(&self, record: &Launch, kind: &str) -> Result> { let args = if kind == "container" { - strings(&["ps", "-aq"]) + strings(&["ps", "-a", "--format", "{{.Names}}"]) } else { - strings(&[kind, "ls", "-q"]) + strings(&[kind, "ls", "--format", "{{.Name}}"]) }; - let ids = self.call(&args).await?; - if ids.is_empty() { + let listed = self.call(&args).await?; + let names: Vec<_> = listed + .lines() + .filter(|name| { + if kind == "network" { + *name == record.resource_name(kind, 0) + } else { + (1..=record.nodes).any(|index| *name == record.resource_name(kind, index)) + } + }) + .collect(); + if names.is_empty() { return Ok(Vec::new()); } + // Other fixtures may remove their resources between list and inspect. + // Inspect only this receipt's exact names, then verify their ownership. let mut args = strings(&[kind, "inspect"]); - args.extend(ids.lines().map(String::from)); - if args.len() > 4098 { - return Err(invalid("Docker resource inventory exceeds bound")); - } + args.extend(names.into_iter().map(String::from)); serde_json::from_str(&self.call(&args).await?) .map_err(|_| invalid("invalid Docker resource observation")) } @@ -99,7 +108,7 @@ impl Docker { } pub async fn ensure(&self, record: &Launch, kind: &str, name: &str) -> Result { if let Some(item) = self - .objects(kind) + .objects(record, kind) .await? .into_iter() .find(|item| object_name(item) == name) diff --git a/app/crowdb-cli/src/commands/cluster/local/launch.rs b/app/crowdb-cli/src/commands/cluster/local/launch.rs index 664205ef6..3582497c8 100644 --- a/app/crowdb-cli/src/commands/cluster/local/launch.rs +++ b/app/crowdb-cli/src/commands/cluster/local/launch.rs @@ -41,7 +41,7 @@ pub(super) async fn accepted(docker: &Docker, cluster: &str, args: &StartArgs) - return Err(invalid("local start requires the Linux amd64 CROWDB node image")); } let record = Launch { - version: 1, + version: 2, cluster: cluster.into(), owner: uuid::Uuid::new_v4().to_string(), runtime: docker.identity().await?, @@ -67,7 +67,7 @@ pub(super) async fn run(docker: &Docker, record: &Launch) -> Result<()> { if bridge["Driver"] != "bridge" { return Err(invalid("local cluster requires its owned bridge")); } - let containers = docker.objects("container").await?; + let containers = docker.objects(record, "container").await?; for index in 1..=record.nodes { let name = record.resource_name("container", index); let volume = record.resource_name("volume", index); @@ -111,10 +111,13 @@ fn arguments(record: &Launch, index: u8) -> Vec { &format!("type=volume,source={volume},target=/opt/crowdb/data"), ])); let base = record.port_base + u16::from(index - 1) * 10; - for (offset, protocol) in [(0_u16, ""), (1, "S3"), (2, "ICEBERG"), (3, "DATASET")] { + for ((offset, protocol), internal) in [(0_u16, ""), (1, "S3"), (2, "ICEBERG"), (3, "DATASET")] + .into_iter() + .zip(record.container_ports()) + { args.extend(strings(&[ "-p", - &format!("127.0.0.1:{}:{}", base + offset, 9090 + offset), + &format!("127.0.0.1:{}:{internal}", base + offset), ])); if !protocol.is_empty() { args.extend(strings(&[ @@ -137,9 +140,9 @@ pub(super) fn verify(item: &Value, record: &Launch, index: u8) -> Result<()> { require_owned(item, record, "container")?; let base = record.port_base + u16::from(index - 1) * 10; let mut ports = serde_json::Map::new(); - for offset in 0..4 { + for (offset, internal) in (0_u16..).zip(record.container_ports()) { ports.insert( - format!("{}/tcp", 9090 + offset), + format!("{internal}/tcp"), json!([{"HostIp":"127.0.0.1", "HostPort":(base+offset).to_string()}]), ); } diff --git a/app/crowdb-cli/src/commands/cluster/local/state.rs b/app/crowdb-cli/src/commands/cluster/local/state.rs index 694db0da8..20769bb0c 100644 --- a/app/crowdb-cli/src/commands/cluster/local/state.rs +++ b/app/crowdb-cli/src/commands/cluster/local/state.rs @@ -152,7 +152,7 @@ impl Launch { pub fn validate(&self) -> Result<()> { let owner = uuid::Uuid::parse_str(&self.owner).map_err(|_| invalid("invalid cluster ownership token"))?; - if self.version != 1 + if !matches!(self.version, 1 | 2) || !name_valid(&self.cluster) || owner.is_nil() || owner.to_string() != self.owner @@ -182,6 +182,18 @@ impl Launch { .map(|index| format!("http://127.0.0.1:{}", self.port_base + u16::from(index) * 10)) .collect() } + pub fn container_ports(&self) -> [u16; 4] { + if self.version == 1 { + [9090, 9091, 9092, 9093] + } else { + [ + 9090, + crowdb_protocol::ACCESS_SERVER_HTTP_BASE, + crowdb_protocol::ACCESS_SERVER_ICEBERG_HTTP_BASE, + crowdb_protocol::ACCESS_SERVER_DATASET_HTTP_BASE, + ] + } + } pub fn resource_name(&self, kind: &str, index: u8) -> String { let base = format!("crowdb-{}", self.cluster); if kind == "network" { diff --git a/app/crowdb-cli/tests/cluster_lifecycle_test.rs b/app/crowdb-cli/tests/cluster_lifecycle_test.rs index 27a966722..c871e83ab 100644 --- a/app/crowdb-cli/tests/cluster_lifecycle_test.rs +++ b/app/crowdb-cli/tests/cluster_lifecycle_test.rs @@ -144,6 +144,39 @@ async fn serve(cluster: &str, nodes: u8, ready: bool) -> (u16, tokio::task::Join ) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn concurrent_foreign_resource_removal_does_not_break_selected_cluster_replay() { + let test = TestDocker::new(); + let (port, server) = serve("local", 1, true).await; + let mut state = test.state(); + state["disappearing"] = json!({ + "container": ["crowdb-foreign-1"], + "volume": ["crowdb-foreign-1-data"], + "network": ["crowdb-foreign"], + }); + test.save(&state); + test.run( + None, + &["start", "--port-base", &port.to_string(), "--wait", "0"], + 0, + ); + let receipt = fs::read(test.receipt("local")).unwrap(); + test.run(None, &["stop"], 0); + test.run(None, &["start", "--wait", "0"], 0); + assert_eq!(fs::read(test.receipt("local")).unwrap(), receipt); + assert_eq!(test.state()["container"].as_object().unwrap().len(), 1); + assert!(test.state()["calls"].as_array().unwrap().iter().all(|args| { + args[1] != "inspect" + || args + .as_array() + .unwrap() + .iter() + .skip(2) + .all(|name| !name.as_str().unwrap().starts_with("crowdb-foreign")) + })); + server.abort(); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn default_single_node_start_seals_identity_and_resumes_without_duplicate_resources() { let test = TestDocker::new(); @@ -154,6 +187,15 @@ async fn default_single_node_start_seals_identity_and_resumes_without_duplicate_ 0, ); let original = fs::read(test.receipt("local")).unwrap(); + let accepted: Value = serde_json::from_slice(&original).unwrap(); + assert_eq!(accepted["version"], 2); + for (internal, offset) in [(9090, 0), (16000, 1), (17000, 2), (18000, 3)] { + assert_eq!( + test.state()["container"]["crowdb-local-1"]["HostConfig"]["PortBindings"] + [format!("{internal}/tcp")][0]["HostPort"], + (port + offset).to_string() + ); + } let binding = test.root.path().join("persistent/clusters/local/target.json"); let target: Value = serde_json::from_slice(&fs::read(binding).unwrap()).unwrap(); assert_eq!(target["cluster_id"], uuid::Uuid::from_u128(1).to_string()); @@ -181,6 +223,47 @@ async fn default_single_node_start_seals_identity_and_resumes_without_duplicate_ server.abort(); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn legacy_receipts_retain_protocol_ports_on_resume_and_recreation() { + let test = TestDocker::new(); + let (port, server) = serve("local", 1, true).await; + test.run( + None, + &["start", "--port-base", &port.to_string(), "--wait", "0"], + 0, + ); + let path = test.receipt("local"); + let mut legacy: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + legacy["version"] = json!(1); + let accepted = serde_json::to_vec(&legacy).unwrap(); + fs::write(&path, &accepted).unwrap(); + let ports: serde_json::Map<_, _> = (0_u16..4) + .map(|offset| { + ( + format!("{}/tcp", 9090 + offset), + json!([{"HostIp":"127.0.0.1","HostPort":(port+offset).to_string()}]), + ) + }) + .collect(); + let mut state = test.state(); + state["container"]["crowdb-local-1"]["HostConfig"]["PortBindings"] = json!(ports); + test.save(&state); + test.run(None, &["stop"], 0); + test.run(None, &["start", "--wait", "0"], 0); + assert_eq!(fs::read(&path).unwrap(), accepted); + let mut state = test.state(); + state["container"] = json!({}); + test.save(&state); + test.run(None, &["start", "--wait", "0"], 0); + assert_eq!(fs::read(&path).unwrap(), accepted); + assert_eq!( + test.state()["container"]["crowdb-local-1"]["HostConfig"]["PortBindings"], + json!(ports) + ); + assert_eq!(test.state()["volume"].as_object().unwrap().len(), 1); + server.abort(); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn three_node_partial_readiness_keeps_resumable_inputs_and_stop_is_scoped() { let test = TestDocker::new(); diff --git a/app/crowdb-cli/tests/common/local_docker.py b/app/crowdb-cli/tests/common/local_docker.py index 74681286b..20a60ea5e 100644 --- a/app/crowdb-cli/tests/common/local_docker.py +++ b/app/crowdb-cli/tests/common/local_docker.py @@ -31,9 +31,11 @@ def one(flag): result = [{"Id": "sha256:" + "a" * 64, "Os": "linux", "Architecture": "amd64", "Config": {"Labels": {"org.opencontainers.image.source": "https://github.com/buzzcrow/crowdb"}}}] elif args[:2] == ["ps", "-aq"]: - result = "\n".join(state["container"]) + result = "\n".join([*state["container"], *state.get("disappearing", {}).get("container", [])]) +elif args[:3] == ["ps", "-a", "--format"]: + result = "\n".join([*state["container"], *state.get("disappearing", {}).get("container", [])]) elif len(args) > 1 and args[1] == "ls": - result = "\n".join(state[args[0]]) + result = "\n".join([*state[args[0]], *state.get("disappearing", {}).get(args[0], [])]) elif len(args) > 1 and args[1] == "inspect": result = [state[args[0]][name] for name in args[2:]] elif len(args) > 1 and args[1] == "create": diff --git a/app/crowdb-diskdb/Cargo.toml b/app/crowdb-diskdb/Cargo.toml index 3f0451ab8..546028912 100644 --- a/app/crowdb-diskdb/Cargo.toml +++ b/app/crowdb-diskdb/Cargo.toml @@ -167,3 +167,7 @@ path = "tests/online_activation_test.rs" [[test]] name = "compaction_schedule_test" path = "tests/compaction_schedule_test.rs" + +[[test]] +name = "journal_gap_recovery_test" +path = "tests/journal_gap_recovery_test.rs" diff --git a/app/crowdb-diskdb/tests/common/journal_recovery.rs b/app/crowdb-diskdb/tests/common/journal_recovery.rs new file mode 100644 index 000000000..08b481885 --- /dev/null +++ b/app/crowdb-diskdb/tests/common/journal_recovery.rs @@ -0,0 +1,86 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use bytes::Bytes; +use crowdb_diskdb::model::zone::DdbZone; +use crowdb_kv::cluster::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_kv::kv::{Batch, BatchOp, CrowdbTreeConfig, CrowdbTreeEngine, KVEngine, Op}; +use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_kv::wal::{index::SegmentIndex, record::WALRecord, replay::ReplayResult}; +use crowdb_protocol::common::{ChunkId, DiskId}; +use crowdb_protocol::diskdb::rpc::BusyBlockValue; +use crowdb_protocol::key::{BinaryKey, BusyBlockKey, ZoneKey}; +use crowdb_protocol::ZoneValueExt; + +pub struct TestReclaimedJournal; + +impl TestReclaimedJournal { + pub async fn replica(disk_id: DiskId) -> PxLocalReplica { + let baseline = DdbZone::new(disk_id, 0, 1, 128); + baseline + .snapshot_slot + .store(1, std::sync::atomic::Ordering::Release); + let zone_key = ZoneKey { + disk_id, + zone_index: 0, + } + .to_bytes(); + let busy_key = BusyBlockKey { + disk_id, + zone_index: 0, + unit_offset: 7, + } + .to_bytes(); + let busy = BusyBlockValue { + unit_count: 1, + owner_chunk: Some(ChunkId { high: 30, low: 40 }), + unit_size: 1024 * 1024, + allocation_ts: 100, + ..Default::default() + }; + let engine = CrowdbTreeEngine::open(&CrowdbTreeConfig::default()).unwrap(); + for (slot, key, value) in [ + (1, zone_key, baseline.to_zone_value().to_bytes()), + (2, busy_key, bincode::serialize(&busy).unwrap()), + ] { + engine + .apply( + slot, + &Batch { + ops: vec![BatchOp { + key: key.into(), + op: Op::Put(value.into()), + }], + }, + ) + .await + .unwrap(); + } + engine.noop(3); + engine.handle().flush().unwrap(); + assert_eq!(engine.resume_from_slot(), 3); + let replay = ReplayResult { + records: vec![WALRecord::from_accepted( + 1, + &PxLogEntry { + slot: 3, + ballot: PxBallot::new(0, 1), + term: 1, + payload: Bytes::new(), + }, + )], + index: SegmentIndex::new(), + max_segment_id: 1, + current_term: 1, + voted_for: None, + }; + PxLocalReplica::restore_from_replay_with_engine( + 1, + PxLocalReplicaRole::Leader, + &replay, + Box::new(engine), + ) + .await + .unwrap() + } +} diff --git a/app/crowdb-diskdb/tests/journal_gap_recovery_test.rs b/app/crowdb-diskdb/tests/journal_gap_recovery_test.rs new file mode 100644 index 000000000..bf46047f1 --- /dev/null +++ b/app/crowdb-diskdb/tests/journal_gap_recovery_test.rs @@ -0,0 +1,70 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_diskdb::{ddb_kv_client::DdbKvClient, recovery::ZoneLoader}; +use crowdb_kv::cluster::group::PxGroup; +use crowdb_kv::cluster::kv_server::KvServer; +use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_protocol::common::{DiskId, HwStatus}; +use crowdb_protocol::diskdb::rpc::DiskValue; +use std::sync::Arc; + +#[path = "common/journal_recovery.rs"] +mod journal_recovery; + +#[tokio::test] +async fn restart_with_reclaimed_journal_rebuilds_busy_units_before_allocating() { + let disk_id = DiskId { high: 10, low: 20 }; + let restored = journal_recovery::TestReclaimedJournal::replica(disk_id).await; + let store = Arc::new(PxKvStore::new(0, "127.0.0.1:0".parse().unwrap())); + store.add_group(PxGroup::new( + 0, + PxLocalReplica::new(1, PxLocalReplicaRole::Leader), + )); + store.add_group(PxGroup::new(1, restored)); + store.start().await.unwrap(); + let client = CrowdbKvClient::new(ClientConfig::new(vec![])); + let endpoint = store.listen_addr().unwrap().to_string(); + for group in [0, 1] { + client.seed_leader(0, group, endpoint.clone()); + } + let loader = ZoneLoader::new(Arc::new(DdbKvClient::new(client)), 1); + let result = loader + .load_disk_group( + 1, + 1, + 1, + (0, 1), + &[( + disk_id, + DiskValue { + capacity_units: 128, + zone_size_units: 128, + unit_size_bytes: 1024 * 1024, + zone_count: 1, + status: HwStatus::Up as i32, + ..Default::default() + }, + )], + 1, + ) + .await; + store.stop(); + let group = result.unwrap(); + group.set_status(HwStatus::Up); + let disks = group.disk_snapshot(); + let zones = disks[0].zones.load(); + assert!( + zones[0].usage_bits.is_set(7), + "reclaimed journal must not erase a durable allocation" + ); + assert_eq!(zones[0].usage_bits.count_set(), 1); + for _ in 0..16 { + let (_, _, allocated) = group.allocate_block(1, &[], 100, 1).unwrap(); + assert_ne!( + allocated.unit_offset, 7, + "live tree extent must never be reallocated" + ); + } +} diff --git a/app/crowdb-kv-server/Cargo.toml b/app/crowdb-kv-server/Cargo.toml index f06ee6a43..06389f473 100644 --- a/app/crowdb-kv-server/Cargo.toml +++ b/app/crowdb-kv-server/Cargo.toml @@ -39,6 +39,7 @@ sha2 = "0.10" uuid = { version = "1", features = ["v4"] } [dev-dependencies] +crowdb-kv = { path = "../../lib/crowdb-kv", features = ["test-util"] } reqwest = { version = "0.12", features = ["json"] } crowdb-rpc-ffi = { path = "../../lib/crowdb-rpc/ffi", features = ["test-util"] } crowdb-test-harness = { path = "../../lib/crowdb-test-harness" } diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv.rs b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv.rs index 193b3ca19..7c757f980 100644 --- a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv.rs +++ b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv.rs @@ -25,6 +25,7 @@ use super::{DomainMonitorDriver, DomainMonitorFuture}; mod balance; mod catalog; mod grants; +mod liveness; pub struct ChunkKvRangeMonitorDriver; @@ -59,8 +60,9 @@ impl DomainMonitorDriver for ChunkKvRangeMonitorDriver { if catalog::refresh_endpoints(control, descriptor).await? { return Ok(()); } - plan_dead_owner_transfer(control, descriptor).await?; - advance_dead_owner_exclusion(control, descriptor).await?; + let live_processes = liveness::live_processes(control).await?; + plan_dead_owner_transfer(control, descriptor, &live_processes).await?; + advance_dead_owner_exclusion(control, descriptor, &live_processes).await?; balance::plan(control, descriptor).await?; authorize_live_source_fences(control, descriptor).await?; publish_ready_transitions(control).await?; @@ -73,6 +75,7 @@ impl DomainMonitorDriver for ChunkKvRangeMonitorDriver { async fn plan_dead_owner_transfer( control: &Group0ControlPlane, descriptor: &DomainMonitorDescriptor, + live_processes: &HashSet, ) -> Result<(), String> { if descriptor.failure_policy != DomainFailurePolicy::AutomaticSharedStorage { return Ok(()); @@ -124,6 +127,7 @@ async fn plan_dead_owner_transfer( }); for entry in catalog.pages.iter().flat_map(|page| &page.entries) { if entry.state != ChunkKvRangeCatalogPartitionState::Serving + || live_processes.contains(&entry.owner.instance_id) || transitioning_partitions.contains(&entry.partition_id) || busy_owners.contains(&entry.owner.instance_id) || instances @@ -215,6 +219,7 @@ async fn plan_dead_owner_transfer( async fn advance_dead_owner_exclusion( control: &Group0ControlPlane, descriptor: &DomainMonitorDescriptor, + live_processes: &HashSet, ) -> Result<(), String> { let now_ms = wall_time_ms(); let dead_before = now_ms.saturating_sub(descriptor.dead_after_ms); @@ -224,6 +229,7 @@ async fn advance_dead_owner_exclusion( transition.phase, TransferPhase::TargetPrepared | TransferPhase::AwaitingFence ) || transition.release_proof.is_some() + || live_processes.contains(&transition.source.instance_id) || instances .get(&transition.source.instance_id) .is_some_and(|instance| instance.last_heartbeat_ms >= dead_before) diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance.rs b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance.rs index eaee82913..61a3700f3 100644 --- a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance.rs +++ b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance.rs @@ -8,8 +8,8 @@ use std::collections::{HashMap, HashSet}; use bytes::Bytes; use crowdb_kv::cluster::group_operations::KvGroupOperationError; use crowdb_protocol::chunk_kv::{ - ChunkKvRangeBalancePolicy, ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogPartitionState, - DomainMonitorDescriptor, Id128, OwnerDescriptor, SplitPhase, TransferPhase, TransferTransition, + ChunkKvRangeBalancePolicy, ChunkKvRangeCatalogEntry, DomainMonitorDescriptor, Id128, OwnerDescriptor, + SplitPhase, TransferPhase, TransferTransition, }; use crowdb_protocol::common::{ChunkKvExtra, ChunkKvPartitionLoad, InstanceValue}; use crowdb_protocol::key::{ChunkKvSplitKey, ChunkKvTransferKey, TextKey}; @@ -21,6 +21,7 @@ use super::{ catalog, operation_error, read_instances, read_splits, read_transfers, transfer_id, wall_time_ms, }; +mod materialization; mod observation; mod selection; mod split; @@ -50,31 +51,8 @@ pub async fn plan(control: &Group0ControlPlane, descriptor: &DomainMonitorDescri if state.healthy.is_empty() { return Ok(()); } - let entries: Vec<_> = catalog - .pages - .iter() - .flat_map(|page| page.entries.iter()) - .collect(); - if let Some(entry) = entries.iter().copied().find(|entry| { - entry.state == ChunkKvRangeCatalogPartitionState::Serving - && entry.artifact.tail_overlay.is_some() - && partition_load(&state, entry).is_some_and(|load| load.independently_recoverable) - }) { - let committed_transfer = read_transfers(control).await?.into_iter().any(|(transition, _)| { - transition.phase == TransferPhase::CatalogCommitted - && entry.transition_id == Some(transition.transition_id) - && entry.partition_id == transition.partition_id - && entry.range == transition.range - && entry.owner.instance_id == transition.target.instance_id - && entry.owner_epoch == transition.target_epoch - && entry.artifact == transition.target_artifact - }); - if committed_transfer { - catalog::publish_materialized_transfer(control, entry.partition_id).await?; - } else { - catalog::publish_materialized_partition(control, entry.partition_id).await?; - } - // Materialization only releases an immutable parent-stream overlay. + if materialization::reconcile(control, &catalog, &state).await? { + // Materialization releases a completed marker or immutable overlay. // It does not affect request routing, so immediately plan from the // refreshed catalog instead of inserting a control-plane idle cycle // before the next independent split. diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/materialization.rs b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/materialization.rs new file mode 100644 index 000000000..35debb5cf --- /dev/null +++ b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/materialization.rs @@ -0,0 +1,63 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Release completed handoff markers after current-owner recovery. + +use crowdb_protocol::chunk_kv::{ + ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogPartitionState, TransferPhase, TransferTransition, +}; + +use crate::group0_control_plane::Group0ControlPlane; + +use super::{catalog, partition_load, read_transfers, PlanningState}; + +pub(super) async fn reconcile( + control: &Group0ControlPlane, + catalog: &catalog::VersionedRangeCatalog, + state: &PlanningState, +) -> Result { + let candidates: Vec<_> = catalog + .pages + .iter() + .flat_map(|page| page.entries.iter()) + .filter(|entry| independently_recovered(entry, state)) + .collect(); + if candidates.is_empty() { + return Ok(false); + } + let transfers = read_transfers(control).await?; + for entry in candidates { + if transfers + .iter() + .any(|(transition, _)| committed_target(entry, transition)) + { + catalog::publish_materialized_transfer(control, entry).await?; + return Ok(true); + } + if entry.artifact.tail_overlay.is_some() { + catalog::publish_materialized_partition(control, entry.partition_id).await?; + return Ok(true); + } + } + Ok(false) +} + +fn independently_recovered(entry: &ChunkKvRangeCatalogEntry, state: &PlanningState) -> bool { + entry.state == ChunkKvRangeCatalogPartitionState::Serving + && (entry.transition_id.is_some() || entry.artifact.tail_overlay.is_some()) + && state + .hosted_epochs + .get(&(entry.owner.instance_id, entry.partition_id)) + == Some(&entry.owner_epoch) + && partition_load(state, entry).is_some_and(|load| load.independently_recoverable) +} + +fn committed_target(entry: &ChunkKvRangeCatalogEntry, transition: &TransferTransition) -> bool { + transition.phase == TransferPhase::CatalogCommitted + && entry.transition_id == Some(transition.transition_id) + && entry.partition_id == transition.partition_id + && entry.range == transition.range + && entry.owner.instance_id == transition.target.instance_id + && entry.owner_epoch == transition.target_epoch + && entry.artifact == transition.target_artifact +} diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/catalog.rs b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/catalog.rs index 065965e33..c340cc986 100644 --- a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/catalog.rs +++ b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/catalog.rs @@ -10,7 +10,9 @@ use crowdb_protocol::chunk_kv::{ ChunkKvRangeCatalogPageRef, ChunkKvRangeCatalogPartitionState, SplitPhase, SplitTransition, TransferPhase, TransferTransition, }; -use crowdb_protocol::key::{ChunkKvRangeCatalogHeadKey, ChunkKvRangeCatalogPageKey, TextKey}; +use crowdb_protocol::key::{ + ChunkKvRangeCatalogHeadKey, ChunkKvRangeCatalogPageKey, ChunkKvSplitKey, TextKey, +}; use crate::group0_control_plane::Group0ControlPlane; @@ -160,13 +162,57 @@ pub async fn publish_materialized_partition( let transition_id = current .transition_id .ok_or_else(|| "materialized split child has no transition identity".to_string())?; + validate_materialized_split(control, &catalog.pages, transition_id).await?; let (head, pages) = release_materialized_split(catalog.head, catalog.pages, transition_id, partition_id)?; publish(control, head, pages, catalog.head_revision).await } +async fn validate_materialized_split( + control: &Group0ControlPlane, + pages: &[ChunkKvRangeCatalogPage], + transition_id: crowdb_protocol::chunk_kv::Id128, +) -> Result<(), String> { + let path = ChunkKvSplitKey { transition_id }.to_path(); + let read = control + .get(path.as_bytes()) + .await + .map_err(|error| error.to_string())?; + let bytes = read.value.ok_or("materialized split transition is absent")?; + let transition: SplitTransition = serde_json::from_slice(&bytes).map_err(|error| error.to_string())?; + transition.validate().map_err(|error| error.to_string())?; + if transition.transition_id != transition_id + || !matches!( + transition.phase, + SplitPhase::ChildPrepared | SplitPhase::CatalogCommitted + ) + { + return Err("materialized split transition is not published".into()); + } + let desired = split_entries(&transition); + for entry in pages.iter().flat_map(|page| &page.entries) { + if entry.transition_id == Some(transition_id) + && !desired + .iter() + .any(|wanted| published_assignment_matches(entry, wanted)) + { + return Err("materialized split assignment differs from its published transition".into()); + } + } + // Once the split commit is durable, lease-excluded recovery may supersede + // either writer independently. Only the remaining exact split assignments + // own this overlay and marker; a successor assignment must stay untouched. + // Before that status write, both exact successors must still prove publication. + if transition.phase != SplitPhase::CatalogCommitted + && !publication_complete(pages, &desired, transition_id)? + { + return Err("materialized split must retain both uncommitted published assignments".into()); + } + Ok(()) +} + pub async fn publish_materialized_transfer( control: &Group0ControlPlane, - partition_id: crowdb_protocol::chunk_kv::Id128, + expected: &ChunkKvRangeCatalogEntry, ) -> Result { let catalog = load_current(control) .await? @@ -175,18 +221,21 @@ pub async fn publish_materialized_transfer( .pages .iter() .flat_map(|page| &page.entries) - .find(|entry| entry.partition_id == partition_id) + .find(|entry| entry.partition_id == expected.partition_id) else { return Err("materialized transfer target is absent from the catalog".into()); }; if current.artifact.tail_overlay.is_none() && current.transition_id.is_none() { return Ok(catalog.head.generation); } + if current != expected { + return Err("materialized transfer assignment changed before publication".into()); + } let transition_id = current .transition_id .ok_or_else(|| "materialized transfer target has no transition identity".to_string())?; let (head, pages) = - release_materialized_transfer(catalog.head, catalog.pages, transition_id, partition_id)?; + release_materialized_transfer(catalog.head, catalog.pages, transition_id, expected.partition_id)?; publish(control, head, pages, catalog.head_revision).await } @@ -208,7 +257,7 @@ fn release_materialized_transfer( continue; } matched += 1; - if entry.partition_id != partition_id || entry.artifact.tail_overlay.is_none() { + if entry.partition_id != partition_id { return Err("materialized transfer target identity is inconsistent".into()); } entry.artifact.tail_overlay = None; @@ -274,8 +323,8 @@ fn release_materialized_split( page.generation = generation; } } - if matched != 2 || !partition_matched { - return Err("materialized split must match one partition in an exact writer pair".into()); + if !(1..=2).contains(&matched) || !partition_matched { + return Err("materialized split must match a remaining published partition".into()); } let all_materialized = pages .iter() diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/grants.rs b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/grants.rs index e8ee91fda..aa95956f9 100644 --- a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/grants.rs +++ b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/grants.rs @@ -3,7 +3,7 @@ //! Grant publication serialized with process replacement and renewal. -use super::{catalog, operation_error, wall_time_ms}; +use super::{catalog, operation_error, read_transfers, wall_time_ms}; use crate::group0_control_plane::Group0ControlPlane; use bytes::Bytes; use crowdb_kv::cluster::group_operations::KvGroupMutation; @@ -13,7 +13,7 @@ use crowdb_protocol::{ common::InstanceValue, key::{InstanceKey, ServingGrantKey, StreamProcessLeaseKey, TextKey}, }; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; struct ReadyInstance { process_epoch: u64, @@ -28,13 +28,26 @@ pub(super) async fn issue( return Ok(()); }; let ready = read_ready(control, descriptor).await?; - let mut assignments: HashMap> = HashMap::new(); + let released: HashSet<_> = read_transfers(control) + .await? + .into_iter() + .filter(|(transition, _)| transition.release_proof.is_some()) + .map(|(transition, _)| { + ( + transition.source.instance_id, + transition.partition_id, + transition.source_epoch, + ) + }) + .collect(); + let mut assignments: BTreeMap> = BTreeMap::new(); for entry in catalog.pages.iter().flat_map(|page| &page.entries) { let assignment = ServingAssignment { partition_id: entry.partition_id, owner_epoch: entry.owner_epoch, }; if entry.state == ChunkKvRangeCatalogPartitionState::Serving + && !released.contains(&(entry.owner.instance_id, entry.partition_id, entry.owner_epoch)) && ready .get(&entry.owner.instance_id) .is_some_and(|instance| instance.assignments.contains(&assignment)) @@ -45,8 +58,9 @@ pub(super) async fn issue( .push(assignment); } } + let mut failures = Vec::new(); for (instance_id, assignments) in assignments { - publish( + if let Err(error) = publish( control, descriptor, instance_id, @@ -54,9 +68,19 @@ pub(super) async fn issue( catalog.head.generation, assignments, ) - .await?; + .await + { + failures.push(format!("instance {instance_id}: {error}")); + } + } + if failures.is_empty() { + Ok(()) + } else { + Err(format!( + "serving grant publication failed: {}", + failures.join("; ") + )) } - Ok(()) } async fn publish( diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/liveness.rs b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/liveness.rs new file mode 100644 index 000000000..7b8590863 --- /dev/null +++ b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/liveness.rs @@ -0,0 +1,36 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Independent process renewal remains visible while partitions replay. + +use super::{operation_error, wall_time_ms, Group0ControlPlane}; +use bytes::Bytes; +use crowdb_protocol::{ + chunk_stream::{StreamProcessLease, StreamPurpose}, + key::{StreamProcessLeaseKey, TextKey}, +}; +use std::collections::HashSet; + +pub(super) async fn live_processes(control: &Group0ControlPlane) -> Result, String> { + let rows = control + .scan_all_prefix(Bytes::from(StreamProcessLeaseKey::prefix_all()), 256) + .await + .map_err(|error| operation_error(&error))?; + let now = wall_time_ms(); + let mut live = HashSet::new(); + for row in rows { + let lease: StreamProcessLease = serde_json::from_slice(&row.value).map_err(|e| e.to_string())?; + lease.validate().map_err(|e| e.to_string())?; + let key = StreamProcessLeaseKey { + purpose: lease.identity.purpose, + instance_id: lease.identity.instance_id, + }; + if row.key.as_ref() != key.to_path().as_bytes() { + return Err("stream process lease differs from its key".into()); + } + if lease.identity.purpose == StreamPurpose::Wal && lease.expires_at_ms > now { + live.insert(lease.identity.instance_id); + } + } + Ok(live) +} diff --git a/app/crowdb-kv-server/src/background/domain_monitor/diskdb_placement.rs b/app/crowdb-kv-server/src/background/domain_monitor/diskdb_placement.rs index 293172501..e8660dd5c 100644 --- a/app/crowdb-kv-server/src/background/domain_monitor/diskdb_placement.rs +++ b/app/crowdb-kv-server/src/background/domain_monitor/diskdb_placement.rs @@ -12,6 +12,9 @@ use crowdb_protocol::common::{BindMapValue, InstanceValue, OwnerMapValue}; use crowdb_protocol::key::{BindMapKey, DiskGroupKey, InstanceKey, OwnerMapKey, RackKey, TextKey}; use std::collections::BTreeMap; +const OWNER_LEASE_MS: u64 = 3_600_000; +const OWNER_RENEW_MARGIN_MS: u64 = 60_000; + pub struct DiskdbPlacementMonitorDriver; impl DomainMonitorDriver for DiskdbPlacementMonitorDriver { @@ -44,29 +47,10 @@ async fn reconcile(control: &Group0ControlPlane, policy: &DomainMonitorDescripto .as_millis() .try_into() .unwrap_or(u64::MAX); - let instances = control - .scan_all_prefix(Bytes::from(InstanceKey::text_prefix_for_service("diskdb")), 256) - .await - .map_err(|e| e.to_string())?; - let mut heartbeats = BTreeMap::new(); - let mut loads = BTreeMap::new(); - for row in instances { - let instance: InstanceValue = decode(&row.value)?; - heartbeats.insert(instance.instance_id, instance.last_heartbeat_ms); - // Only the fenced server implementation publishes this capability. - let capable = control - .get(format!("/diskdb/ownership-capability/{}", instance.instance_id).as_bytes()) - .await - .map_err(|e| e.to_string())? - .value - .is_some(); - if capable - && !instance.rpc_endpoint.is_empty() - && now.saturating_sub(instance.last_heartbeat_ms) < policy.suspect_after_ms - { - loads.insert(instance.instance_id, 0usize); - } - } + let LiveInstances { + heartbeats, + mut loads, + } = live_instances(control, policy, now).await?; if loads.is_empty() { return Ok(()); } @@ -97,12 +81,9 @@ async fn reconcile(control: &Group0ControlPlane, policy: &DomainMonitorDescripto } groups.push((key, owner_key, owner.revision, value)); } + let mut renewed = false; for (group, key, revision, owner) in &groups { - let needs_owner = owner.as_ref().map_or(true, |owner| { - heartbeats - .get(&owner.instance_id) - .map_or(true, |last| now.saturating_sub(*last) >= policy.dead_after_ms) - }); + let needs_owner = owner_is_dead(owner.as_ref(), &heartbeats, now, policy.dead_after_ms); if needs_owner { if !has_data_binding(control, group).await? { continue; @@ -110,8 +91,15 @@ async fn reconcile(control: &Group0ControlPlane, policy: &DomainMonitorDescripto let target = least_loaded(&loads); assign(control, key, *revision, target, now).await?; *loads.get_mut(&target).expect("selected live target") += 1; + } else if renew_live_owner(control, key, *revision, owner.as_ref(), &loads, now).await? { + renewed = true; } } + // Renewal advances the map revision. Balance from a fresh snapshot on the + // next tick instead of attempting a move with that old generation. + if renewed { + return Ok(()); + } // Move at most one healthy group per tick. This bounds recovery work and // converges to a count difference of at most one without ping-pong. let target = least_loaded(&loads); @@ -132,6 +120,79 @@ async fn reconcile(control: &Group0ControlPlane, policy: &DomainMonitorDescripto Ok(()) } +fn owner_is_dead( + owner: Option<&OwnerMapValue>, + heartbeats: &BTreeMap, + now: u64, + dead_after_ms: u64, +) -> bool { + owner.map_or(true, |owner| { + heartbeats + .get(&owner.instance_id) + .map_or(true, |last| now.saturating_sub(*last) >= dead_after_ms) + }) +} + +async fn renew_live_owner( + control: &Group0ControlPlane, + key: &str, + revision: u64, + owner: Option<&OwnerMapValue>, + loads: &BTreeMap, + now: u64, +) -> Result { + let Some(owner) = owner.filter(|owner| { + loads.contains_key(&owner.instance_id) + && owner.lease_expiry_ms <= now.saturating_add(OWNER_RENEW_MARGIN_MS) + }) else { + return Ok(false); + }; + let mut value = owner.clone(); + value.lease_expiry_ms = now.saturating_add(OWNER_LEASE_MS); + publish_owner(control, key, revision, value).await?; + Ok(true) +} + +struct LiveInstances { + heartbeats: BTreeMap, + loads: BTreeMap, +} + +async fn live_instances( + control: &Group0ControlPlane, + policy: &DomainMonitorDescriptor, + now: u64, +) -> Result { + let instances = control + .scan_all_prefix(Bytes::from(InstanceKey::text_prefix_for_service("diskdb")), 256) + .await + .map_err(|error| error.to_string())?; + let mut result = LiveInstances { + heartbeats: BTreeMap::new(), + loads: BTreeMap::new(), + }; + for row in instances { + let instance: InstanceValue = decode(&row.value)?; + result + .heartbeats + .insert(instance.instance_id, instance.last_heartbeat_ms); + // Only the fenced server implementation publishes this capability. + let capable = control + .get(format!("/diskdb/ownership-capability/{}", instance.instance_id).as_bytes()) + .await + .map_err(|error| error.to_string())? + .value + .is_some(); + if capable + && !instance.rpc_endpoint.is_empty() + && now.saturating_sub(instance.last_heartbeat_ms) < policy.suspect_after_ms + { + result.loads.insert(instance.instance_id, 0); + } + } + Ok(result) +} + fn least_loaded(loads: &BTreeMap) -> u64 { *loads .iter() @@ -150,8 +211,23 @@ async fn assign( let value = OwnerMapValue { process_epoch: 0, instance_id: instance, - lease_expiry_ms: now.saturating_add(3_600_000), + lease_expiry_ms: now.saturating_add(OWNER_LEASE_MS), }; + publish_owner(control, key, revision, value).await?; + tracing::info!( + owner_key = key, + instance_id = instance, + "DiskGroup manager assigned" + ); + Ok(()) +} + +async fn publish_owner( + control: &Group0ControlPlane, + key: &str, + revision: u64, + value: OwnerMapValue, +) -> Result<(), String> { let placement = OwnerMapKey::from_path(key).map_err(|error| error.to_string())?; let rack_key = RackKey { rack_id: placement.rack_id, @@ -208,11 +284,6 @@ async fn assign( .await .map_err(|error| error.to_string())?; } - tracing::info!( - owner_key = key, - instance_id = instance, - "DiskGroup manager assigned" - ); Ok(()) } diff --git a/app/crowdb-kv-server/src/recovery/startup.rs b/app/crowdb-kv-server/src/recovery/startup.rs index 93e6756b4..932116dc6 100644 --- a/app/crowdb-kv-server/src/recovery/startup.rs +++ b/app/crowdb-kv-server/src/recovery/startup.rs @@ -185,9 +185,15 @@ pub async fn create_group_with_wal( wal_config.wal_record_format = crowdb_kv::wal::WalRecordFormat::TextLine; } wal_config.wal_skip_fsync = config.wal_skip_fsync; - let replay = replay_group(&wal_backend, &wal_config.wal_disks, group_id).await?; - let next_seg = replay.max_segment_id.saturating_add(1).max(1); - let wal = WalEngine::create_with_next_segment_id(wal_backend, wal_config, group_id, next_seg).await?; + let mut replay = replay_group(&wal_backend, &wal_config.wal_disks, group_id).await?; + let wal = WalEngine::create_from_replay( + wal_backend, + wal_config, + group_id, + replay.max_segment_id, + std::mem::take(&mut replay.index), + ) + .await?; let mut local_replica = { let engine = open_crowdb_tree_engine( @@ -198,6 +204,7 @@ pub async fn create_group_with_wal( &config.log_dir, ) .await?; + wal.set_snapshot_slot(engine.resume_from_slot()); PxLocalReplica::restore_from_replay_with_engine(replica_id, initial_role, &replay, engine).await? }; local_replica.set_wal(wal); diff --git a/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test.rs b/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test.rs index 6cd9f43ec..9f9503ab8 100644 --- a/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test.rs +++ b/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test.rs @@ -5,6 +5,10 @@ mod balance_distribution; #[path = "common/balance_planning.rs"] mod balance_planning; +#[path = "chunk_kv_balance_planning_test/committed_transfer.rs"] +mod committed_transfer; +#[path = "chunk_kv_balance_planning_test/recovering_owner.rs"] +mod recovering_owner; use bytes::Bytes; use crowdb_kv::cluster::{group::PxGroup, PxKvStore, PxLocalReplica, PxLocalReplicaRole}; diff --git a/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test/committed_transfer.rs b/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test/committed_transfer.rs new file mode 100644 index 000000000..de925de43 --- /dev/null +++ b/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test/committed_transfer.rs @@ -0,0 +1,143 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; +use crowdb_protocol::chunk_kv::{ + AuthorityReleaseProof, ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage, + OwnerDescriptor, TargetReadinessProof, TransferPhase, TransferReadinessLimits, +}; +use crowdb_protocol::common::InstanceValue; +use crowdb_protocol::key::{ChunkKvRangeCatalogHeadKey, ChunkKvRangeCatalogPageKey, InstanceKey}; + +#[tokio::test] +async fn completed_recovery_does_not_starve_an_idle_owner_of_placement() { + let fixture = TestBalancePlanning::new().await; + seed_committed_recoveries(&fixture.control).await; + fixture.tick().await; + let rows = fixture + .control + .scan_all_prefix(Bytes::from(ChunkKvTransferKey::text_prefix_all()), 16) + .await + .unwrap(); + let planned: Vec = rows + .iter() + .map(|row| serde_json::from_slice(&row.value).unwrap()) + .filter(|transfer: &TransferTransition| transfer.phase == TransferPhase::Planned) + .collect(); + assert_eq!( + planned.len(), + 1, + "completed recovery must permit another placement" + ); + assert_eq!( + (planned[0].source.instance_id, planned[0].target.instance_id), + (1, 2) + ); + assert_eq!(planned[0].partition_id, Id128 { high: 41, low: 1 }); +} + +async fn seed_committed_recoveries(control: &Group0ControlPlane) { + let mut head: ChunkKvRangeCatalogHead = read(control, ChunkKvRangeCatalogHeadKey.to_path()).await; + let mut page: ChunkKvRangeCatalogPage = read( + control, + ChunkKvRangeCatalogPageKey { + generation: head.generation, + page_index: 0, + } + .to_path(), + ) + .await; + let mut records = Vec::new(); + for entry in &mut page.entries { + let transfer = committed_recovery(entry); + transfer.validate().unwrap(); + entry.owner_epoch = transfer.target_epoch; + entry.transition_id = Some(transfer.transition_id); + records.push(record( + ChunkKvTransferKey { + transition_id: transfer.transition_id, + } + .to_path(), + &transfer, + )); + } + page.generation += 1; + page.seal().unwrap(); + head.previous_generation = Some(head.generation); + head.generation = page.generation; + head.pages[0].page_generation = page.generation; + head.pages[0].page_checksum = page.checksum; + head.seal().unwrap(); + head.validate_pages(std::slice::from_ref(&page)).unwrap(); + records.push(record( + ChunkKvRangeCatalogPageKey { + generation: page.generation, + page_index: 0, + } + .to_path(), + &page, + )); + records.push(record(ChunkKvRangeCatalogHeadKey.to_path(), &head)); + let path = InstanceKey { + service: "chunk-kv".into(), + instance_id: 1, + } + .to_path(); + let mut instance: InstanceValue = read(control, path.clone()).await; + for hosted in &mut instance.extra.as_mut().unwrap().chunk_kv.as_mut().unwrap().hosted { + hosted.owner_epoch = 2; + } + records.push(record(path, &instance)); + control.put_batch(records).await.unwrap(); +} + +async fn read(control: &Group0ControlPlane, path: String) -> T { + let row = control.get(path.as_bytes()).await.unwrap(); + serde_json::from_slice(row.value.as_deref().unwrap()).unwrap() +} + +fn record(path: String, value: &T) -> (Bytes, Bytes) { + (Bytes::from(path), Bytes::from(serde_json::to_vec(value).unwrap())) +} + +fn committed_recovery(entry: &ChunkKvRangeCatalogEntry) -> TransferTransition { + let proof = TargetReadinessProof { + target_instance_id: entry.owner.instance_id, + target_epoch: entry.owner_epoch + 1, + artifact: entry.artifact.clone(), + durable_tail: 0, + }; + TransferTransition { + transition_id: Id128 { + high: 99, + low: entry.partition_id.low, + }, + partition_id: entry.partition_id, + range: entry.range.clone(), + source: OwnerDescriptor { + instance_id: 3, + rpc_endpoint: "127.0.0.1:17003".into(), + }, + source_epoch: entry.owner_epoch, + target: entry.owner.clone(), + target_epoch: proof.target_epoch, + artifact: entry.artifact.clone(), + target_artifact: entry.artifact.clone(), + readiness_limits: TransferReadinessLimits { + max_tail_records: 100, + max_tail_bytes: 1_000_000, + max_estimated_catchup_ms: 1_000, + prepare_deadline_ms: 1_000, + forwarding_grace_ms: 1_000, + }, + planned_at_ms: 0, + old_grant_expires_at_ms: 1, + phase: TransferPhase::CatalogCommitted, + release_proof: Some(AuthorityReleaseProof::LeaseExpired { + activation_not_before_ms: 2, + }), + readiness_proof: Some(proof.clone()), + catchup_proof: Some(proof), + failure: None, + } +} diff --git a/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test/recovering_owner.rs b/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test/recovering_owner.rs new file mode 100644 index 000000000..3fb2585a2 --- /dev/null +++ b/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test/recovering_owner.rs @@ -0,0 +1,79 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; +use crowdb_protocol::{ + chunk_stream::{StreamProcessIdentity, StreamProcessLease, StreamPurpose}, + common::InstanceValue, + key::{InstanceKey, StreamProcessLeaseKey}, +}; +use std::time::{SystemTime, UNIX_EPOCH}; + +#[tokio::test] +async fn renewing_recovery_process_is_not_dead_when_discovery_is_stale() { + let fixture = TestBalancePlanning::new().await; + let path = InstanceKey { + service: "chunk-kv".into(), + instance_id: 1, + } + .to_path(); + let row = fixture.control.get(path.as_bytes()).await.unwrap(); + let mut instance: InstanceValue = serde_json::from_slice(row.value.as_deref().unwrap()).unwrap(); + instance.last_heartbeat_ms = 0; + fixture + .control + .put_batch(vec![( + Bytes::from(path), + Bytes::from(serde_json::to_vec(&instance).unwrap()), + )]) + .await + .unwrap(); + let now = u64::try_from(SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis()).unwrap(); + let path = StreamProcessLeaseKey { + purpose: StreamPurpose::Wal, + instance_id: 1, + } + .to_path(); + let mut lease = StreamProcessLease { + identity: StreamProcessIdentity { + purpose: StreamPurpose::Wal, + instance_id: 1, + process_epoch: 2, + boot_nonce: [9; 16], + }, + expires_at_ms: now + 60_000, + prior_exclusion_deadline_ms: now - 1, + grant_sequence: 0, + grant_expires_at_ms: 0, + }; + lease.validate().unwrap(); + for expiry in [now + 60_000, now - 1] { + lease.expires_at_ms = expiry; + fixture + .control + .put_batch(vec![( + Bytes::from(path.clone()), + Bytes::from(serde_json::to_vec(&lease).unwrap()), + )]) + .await + .unwrap(); + fixture.tick().await; + let transfers = fixture + .control + .scan_all_prefix(Bytes::from(ChunkKvTransferKey::text_prefix_all()), 16) + .await + .unwrap(); + if expiry > now { + assert!( + transfers.is_empty(), + "a live recovery boot must retain its assigned WAL" + ); + } else { + assert_eq!( + transfers.len(), + 1, + "expired processes still require dead-owner failover" + ); + } + } +} diff --git a/app/crowdb-kv-server/tests/common/catalog_publication.rs b/app/crowdb-kv-server/tests/common/catalog_publication.rs index 89f108060..5599d7a60 100644 --- a/app/crowdb-kv-server/tests/common/catalog_publication.rs +++ b/app/crowdb-kv-server/tests/common/catalog_publication.rs @@ -169,7 +169,7 @@ impl TestPublication { } } -async fn replace_json(control: &Group0ControlPlane, key: String, value: &T) { +pub(super) async fn replace_json(control: &Group0ControlPlane, key: String, value: &T) { let current = control.get(key.as_bytes()).await.unwrap(); control .compare_and_put( @@ -227,7 +227,7 @@ async fn publish_materialized_catalog( head } -async fn tick(control: &Group0ControlPlane) -> Result<(), String> { +pub(super) async fn tick(control: &Group0ControlPlane) -> Result<(), String> { let mut policy = descriptor(); policy.domain = "chunk-kv".into(); policy.service_registry_name = "chunk-kv".into(); diff --git a/app/crowdb-kv-server/tests/common/grant_isolation.rs b/app/crowdb-kv-server/tests/common/grant_isolation.rs new file mode 100644 index 000000000..d8e2c251a --- /dev/null +++ b/app/crowdb-kv-server/tests/common/grant_isolation.rs @@ -0,0 +1,228 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; +use crowdb_kv_server::background::domain_monitor::ChunkKvRangeMonitorDriver; +use crowdb_protocol::{ + chunk_stream::{StreamProcessLease, StreamPurpose}, + key::StreamProcessLeaseKey, +}; + +struct TestGrantInstances { + store: Arc, + control: Group0ControlPlane, + policy: DomainMonitorDescriptor, +} + +fn lease_path(instance_id: u64) -> String { + StreamProcessLeaseKey { + purpose: StreamPurpose::Wal, + instance_id, + } + .to_path() +} + +impl TestGrantInstances { + async fn start() -> Self { + let (_, store) = registry_with_group_zero(PxLocalReplicaRole::Leader); + let control = Group0ControlPlane::acquire(&store).await.unwrap(); + let mut entries = Vec::new(); + for instance_id in 1..=2 { + let partition_id = Id128 { + high: 91, + low: instance_id, + }; + let endpoint = format!("127.0.0.1:{}", 17000 + instance_id); + entries.push(ChunkKvRangeCatalogEntry { + partition_id, + range: KeyRange { + start: if instance_id == 1 { vec![] } else { b"m".to_vec() }, + end: (instance_id == 1).then(|| b"m".to_vec()), + }, + owner: OwnerDescriptor { + instance_id, + rpc_endpoint: endpoint.clone(), + }, + owner_epoch: 1, + state: ChunkKvRangeCatalogPartitionState::Serving, + artifact: PartitionArtifact { + tree_id: 910 + instance_id, + stream_name: StreamName { + high: 92, + low: instance_id, + }, + tail_overlay: None, + }, + transition_id: None, + }); + test_process::TestProcessLease::seed(&control, instance_id).await; + let mut instance = test_process::TestProcessLease::idle_instance(instance_id, &endpoint); + instance.extra.as_mut().unwrap().chunk_kv.as_mut().unwrap().hosted = vec![HostedPartition { + partition_id, + owner_epoch: 1, + recovering: false, + }]; + put_json( + &control, + InstanceKey { + service: "chunk-kv".into(), + instance_id, + } + .to_path(), + &instance, + ) + .await; + } + Self::seed_catalog(&control, entries).await; + let mut policy = descriptor(); + policy.domain = "chunk-kv".into(); + policy.service_registry_name = "chunk-kv".into(); + policy.failure_policy = DomainFailurePolicy::OperatorOnly; + Self { + store, + control, + policy, + } + } + + async fn seed_catalog(control: &Group0ControlPlane, entries: Vec) { + let mut page = ChunkKvRangeCatalogPage { + generation: 1, + page_index: 0, + entries, + checksum: [0; 32], + }; + page.seal().unwrap(); + let mut head = ChunkKvRangeCatalogHead { + generation: 1, + previous_generation: None, + pages: vec![ChunkKvRangeCatalogPageRef { + page_generation: 1, + page_index: 0, + first_key: vec![], + page_checksum: page.checksum, + }], + checksum: [0; 32], + }; + head.seal().unwrap(); + put_json( + control, + ChunkKvRangeCatalogPageKey { + generation: 1, + page_index: 0, + } + .to_path(), + &page, + ) + .await; + put_json(control, ChunkKvRangeCatalogHeadKey.to_path(), &head).await; + } + + async fn assert_second_grant(&self) { + let path = ServingGrantKey { instance_id: 2 }.to_path(); + let observed = self.control.get(path.as_bytes()).await.unwrap(); + assert!( + observed.value.is_some(), + "one instance's failed grant publication must not skip an independent ready instance" + ); + let grant: ServingGrant = serde_json::from_slice(&observed.value.unwrap()).unwrap(); + let lease_record = self.control.get(lease_path(2).as_bytes()).await.unwrap(); + let lease: StreamProcessLease = serde_json::from_slice(&lease_record.value.unwrap()).unwrap(); + assert_eq!(grant.instance_id, 2); + assert_eq!(grant.process_epoch, 1); + assert_eq!(grant.catalog_generation, 1); + assert_eq!(grant.assignments.len(), 1); + assert_eq!(grant.assignments[0].partition_id, Id128 { high: 91, low: 2 }); + assert_eq!(grant.assignments[0].owner_epoch, 1); + assert_eq!(grant.lease_sequence, lease.grant_sequence); + assert!(lease.grant_expires_at_ms >= grant.expires_at_ms); + assert_eq!(observed.revision, lease_record.revision); + assert!(self + .control + .get(ServingGrantKey { instance_id: 1 }.to_path().as_bytes()) + .await + .unwrap() + .value + .is_none()); + } +} + +async fn until(condition: impl Fn() -> bool) { + tokio::time::timeout(Duration::from_secs(3), async { + while !condition() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn busy_instance_cas_does_not_skip_another_instances_grant() { + let test = TestGrantInstances::start().await; + let group = test.store.get_group(0).unwrap(); + let first_path = lease_path(1); + let current = test.control.get(first_path.as_bytes()).await.unwrap(); + let original_lease = current.value.clone(); + let permits = group.try_acquire_all_inflight_permits(); + assert_eq!(permits.len(), group.inflight_window_size()); + let contender = tokio::spawn({ + let control = test.control.clone(); + let path = first_path.clone(); + async move { + control + .compare_and_put(Bytes::from(path), current.value.unwrap(), current.revision) + .await + } + }); + until(|| group.conditional_write_pending_for_tests(first_path.as_bytes())).await; + let renewal = tokio::spawn({ + let control = test.control.clone(); + let policy = test.policy.clone(); + async move { ChunkKvRangeMonitorDriver::new().tick(&control, &policy).await } + }); + until(|| renewal.is_finished() || group.conditional_write_pending_for_tests(lease_path(2).as_bytes())) + .await; + drop(permits); + let result = tokio::time::timeout(Duration::from_secs(3), renewal) + .await + .unwrap() + .unwrap(); + tokio::time::timeout(Duration::from_secs(3), contender) + .await + .unwrap() + .unwrap() + .unwrap(); + assert!(result.unwrap_err().contains("admission is busy")); + assert_eq!( + test.control.get(first_path.as_bytes()).await.unwrap().value, + original_lease + ); + test.assert_second_grant().await; +} + +#[tokio::test] +async fn invalid_instance_grant_does_not_skip_another_instances_grant() { + let test = TestGrantInstances::start().await; + let path = lease_path(1); + let current = test.control.get(path.as_bytes()).await.unwrap(); + let mut lease: StreamProcessLease = serde_json::from_slice(¤t.value.unwrap()).unwrap(); + lease.grant_sequence = u64::MAX; + lease.grant_expires_at_ms = 1; + lease.validate().unwrap(); + test.control + .compare_and_put( + Bytes::from(path), + Bytes::from(serde_json::to_vec(&lease).unwrap()), + current.revision, + ) + .await + .unwrap(); + let result = ChunkKvRangeMonitorDriver::new() + .tick(&test.control, &test.policy) + .await; + assert!(result.unwrap_err().contains("grant sequence overflowed")); + let retained: StreamProcessLease = get_json(&test.control, &lease_path(1)).await; + assert_eq!(retained, lease); + test.assert_second_grant().await; +} diff --git a/app/crowdb-kv-server/tests/common/grant_process.rs b/app/crowdb-kv-server/tests/common/grant_process.rs index 52304f0d9..830f0c73b 100644 --- a/app/crowdb-kv-server/tests/common/grant_process.rs +++ b/app/crowdb-kv-server/tests/common/grant_process.rs @@ -80,3 +80,39 @@ async fn grant_and_process_expiry_fence_publish_together_and_reject_stale_observ assert_eq!(accepted.grant_sequence, current.lease_sequence); assert!(accepted.grant_expires_at_ms >= current.expires_at_ms); } + +pub(super) async fn assert_returned_source_is_excluded( + control: &Group0ControlPlane, + policy: &DomainMonitorDescriptor, + partition_id: Id128, +) { + // A replacement source can report the old assignment while the catalog + // still names it. Durable exclusion must prevent granting it again. + test_process::TestProcessLease::seed(control, 1).await; + let mut returned = test_process::TestProcessLease::idle_instance(1, "127.0.0.1:17001"); + returned.extra.as_mut().unwrap().chunk_kv.as_mut().unwrap().hosted = vec![HostedPartition { + partition_id, + owner_epoch: 5, + recovering: false, + }]; + put_json( + control, + InstanceKey { + service: "chunk-kv".into(), + instance_id: 1, + } + .to_path(), + &returned, + ) + .await; + ChunkKvRangeMonitorDriver::new() + .tick(control, policy) + .await + .unwrap(); + assert!(control + .get(ServingGrantKey { instance_id: 1 }.to_path().as_bytes()) + .await + .unwrap() + .value + .is_none()); +} diff --git a/app/crowdb-kv-server/tests/common/split_materialization.rs b/app/crowdb-kv-server/tests/common/split_materialization.rs new file mode 100644 index 000000000..4fdb544ec --- /dev/null +++ b/app/crowdb-kv-server/tests/common/split_materialization.rs @@ -0,0 +1,273 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::catalog_publication::{replace_json, tick}; +use super::*; + +#[path = "split_materialization/transfer.rs"] +mod transfer; + +struct TestMaterialization { + control: Group0ControlPlane, + split: SplitTransition, + head: ChunkKvRangeCatalogHead, + page: ChunkKvRangeCatalogPage, +} + +#[tokio::test] +async fn child_materializes_before_the_split_commit_status_is_persisted() { + let (control, prepared) = assert_chunk_kv_split_plan(2, u64::MAX).await; + replace_json( + &control, + ChunkKvSplitKey { + transition_id: prepared.transition_id, + } + .to_path(), + &prepared, + ) + .await; + test_process::TestProcessLease::seed(&control, 1).await; + let (before, original) = current_catalog(&control).await; + observe_partition(&control, &original.entries[1], true).await; + tick(&control).await.unwrap(); + let (head, page) = current_catalog(&control).await; + assert_eq!(head.generation, before.generation + 1); + for (mut expected, actual) in original.entries.into_iter().zip(page.entries) { + expected.artifact.tail_overlay = None; + expected.transition_id = None; + assert_eq!(actual, expected); + } + let committed: SplitTransition = get_json( + &control, + &ChunkKvSplitKey { + transition_id: prepared.transition_id, + } + .to_path(), + ) + .await; + assert_eq!(committed.phase, SplitPhase::CatalogCommitted); +} + +#[tokio::test] +async fn transferred_split_parent_does_not_block_child_materialization_or_grants() { + let fixture = TestMaterialization::transferred_parent().await; + let parent = fixture.page.entries[0].clone(); + fixture.observe_child().await; + tick(&fixture.control).await.unwrap(); + let (head, page) = current_catalog(&fixture.control).await; + assert_eq!(head.generation, fixture.head.generation + 1); + assert_eq!(page.entries[0], parent); + let mut child = fixture.page.entries[1].clone(); + child.artifact.tail_overlay = None; + child.transition_id = None; + assert_eq!(page.entries[1], child); + for entry in &page.entries { + let grant: ServingGrant = get_json( + &fixture.control, + &ServingGrantKey { + instance_id: entry.owner.instance_id, + } + .to_path(), + ) + .await; + assert_eq!(grant.catalog_generation, head.generation); + assert_eq!(grant.assignments.len(), 1); + assert_eq!(grant.assignments[0].partition_id, entry.partition_id); + assert_eq!(grant.assignments[0].owner_epoch, entry.owner_epoch); + } + tick(&fixture.control).await.unwrap(); + assert_eq!(current_catalog(&fixture.control).await.0, head); +} + +#[tokio::test] +async fn split_materialization_rejects_changed_assignment_or_reused_marker() { + for mismatch in 0..3 { + let mut fixture = TestMaterialization::transferred_parent().await; + match mismatch { + 0 => fixture.page.entries[1].owner_epoch += 1, + 1 => fixture.page.entries[1].artifact.tree_id += 1, + 2 => fixture.page.entries[0].transition_id = Some(fixture.split.transition_id), + _ => unreachable!(), + } + fixture.publish_page().await; + fixture.observe_child().await; + let error = tick(&fixture.control) + .await + .expect_err("changed split identity must be rejected"); + assert!(error.contains("materialized split"), "{error}"); + assert_eq!(current_catalog(&fixture.control).await.0, fixture.head); + } +} + +#[tokio::test] +async fn uncommitted_split_still_requires_both_published_assignments() { + let fixture = TestMaterialization::transferred_parent().await; + replace_json( + &fixture.control, + ChunkKvSplitKey { + transition_id: fixture.split.transition_id, + } + .to_path(), + &fixture.split, + ) + .await; + assert_eq!(fixture.split.phase, SplitPhase::ChildPrepared); + fixture.observe_child().await; + assert!(tick(&fixture.control).await.is_err()); + assert_eq!(current_catalog(&fixture.control).await.0, fixture.head); +} + +impl TestMaterialization { + async fn transferred_parent() -> Self { + let (control, split) = assert_chunk_kv_split_plan(2, u64::MAX).await; + let (_, page) = current_catalog(&control).await; + let parent = &page.entries[0]; + let target = OwnerDescriptor { + instance_id: 2, + rpc_endpoint: "127.0.0.1:17002".into(), + }; + let proof = TargetReadinessProof { + target_instance_id: target.instance_id, + target_epoch: parent.owner_epoch + 1, + artifact: parent.artifact.clone(), + durable_tail: 11, + }; + let transfer = TransferTransition { + transition_id: Id128 { high: 97, low: 98 }, + partition_id: parent.partition_id, + range: parent.range.clone(), + source: parent.owner.clone(), + source_epoch: parent.owner_epoch, + target: target.clone(), + target_epoch: proof.target_epoch, + artifact: parent.artifact.clone(), + target_artifact: parent.artifact.clone(), + readiness_limits: TransferReadinessLimits { + max_tail_records: 100, + max_tail_bytes: 1_000_000, + max_estimated_catchup_ms: 1_000, + prepare_deadline_ms: 1_000, + forwarding_grace_ms: 1_000, + }, + planned_at_ms: 0, + old_grant_expires_at_ms: 1, + phase: TransferPhase::TargetReady, + release_proof: Some(AuthorityReleaseProof::LeaseExpired { + activation_not_before_ms: 2, + }), + readiness_proof: Some(proof.clone()), + catchup_proof: Some(proof), + failure: None, + }; + transfer.validate().unwrap(); + put_json( + &control, + ChunkKvTransferKey { + transition_id: transfer.transition_id, + } + .to_path(), + &transfer, + ) + .await; + for instance_id in [1, 2] { + test_process::TestProcessLease::seed(&control, instance_id).await; + } + let mut moved = parent.clone(); + moved.owner = target; + moved.owner_epoch = transfer.target_epoch; + observe_partition(&control, &moved, false).await; + observe_partition(&control, &page.entries[1], false).await; + tick(&control).await.unwrap(); + let (head, page) = current_catalog(&control).await; + assert_eq!(page.entries[0].transition_id, Some(transfer.transition_id)); + assert_eq!(page.entries[1].transition_id, Some(split.transition_id)); + assert_eq!(page.entries[0].owner_epoch, transfer.target_epoch); + Self { + control, + split, + head, + page, + } + } + + async fn observe_child(&self) { + observe_partition(&self.control, &self.page.entries[1], true).await; + } + + async fn publish_page(&mut self) { + self.page.generation += 1; + self.page.seal().unwrap(); + self.head.previous_generation = Some(self.head.generation); + self.head.generation = self.page.generation; + self.head.pages[0].page_generation = self.page.generation; + self.head.pages[0].page_checksum = self.page.checksum; + self.head.seal().unwrap(); + self.head + .validate_pages(std::slice::from_ref(&self.page)) + .unwrap(); + put_json( + &self.control, + ChunkKvRangeCatalogPageKey { + generation: self.page.generation, + page_index: 0, + } + .to_path(), + &self.page, + ) + .await; + replace_json(&self.control, ChunkKvRangeCatalogHeadKey.to_path(), &self.head).await; + } +} + +async fn observe_partition( + control: &Group0ControlPlane, + entry: &ChunkKvRangeCatalogEntry, + independent: bool, +) { + let instance = InstanceValue { + instance_id: entry.owner.instance_id, + rpc_endpoint: entry.owner.rpc_endpoint.clone(), + last_heartbeat_ms: u64::MAX, + extra: Some(ServiceExtra { + chunk_kv: Some(ChunkKvExtra { + process_epoch: 1, + hosted: vec![HostedPartition { + partition_id: entry.partition_id, + owner_epoch: entry.owner_epoch, + recovering: false, + }], + partition_loads: vec![ChunkKvPartitionLoad { + partition_id: entry.partition_id, + independently_recoverable: independent, + ..ChunkKvPartitionLoad::default() + }], + ..ChunkKvExtra::default() + }), + ..ServiceExtra::default() + }), + }; + replace_json( + control, + InstanceKey { + service: "chunk-kv".into(), + instance_id: instance.instance_id, + } + .to_path(), + &instance, + ) + .await; +} + +async fn current_catalog(control: &Group0ControlPlane) -> (ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage) { + let head: ChunkKvRangeCatalogHead = get_json(control, &ChunkKvRangeCatalogHeadKey.to_path()).await; + let page = get_json( + control, + &ChunkKvRangeCatalogPageKey { + generation: head.pages[0].page_generation, + page_index: 0, + } + .to_path(), + ) + .await; + (head, page) +} diff --git a/app/crowdb-kv-server/tests/common/split_materialization/transfer.rs b/app/crowdb-kv-server/tests/common/split_materialization/transfer.rs new file mode 100644 index 000000000..c291da7e0 --- /dev/null +++ b/app/crowdb-kv-server/tests/common/split_materialization/transfer.rs @@ -0,0 +1,75 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; + +#[tokio::test] +async fn committed_transfer_without_overlay_releases_its_catalog_marker() { + let fixture = TestMaterialization::transferred_parent().await; + let parent = &fixture.page.entries[0]; + assert!(parent.artifact.tail_overlay.is_none()); + assert!(parent.transition_id.is_some()); + observe_partition(&fixture.control, parent, true).await; + tick(&fixture.control).await.unwrap(); + + let (head, page) = current_catalog(&fixture.control).await; + let mut expected = parent.clone(); + expected.transition_id = None; + assert_eq!(page.entries[0], expected); + assert_eq!(page.entries[1], fixture.page.entries[1]); + assert_eq!(head.generation, fixture.head.generation + 1); + + let transfer: TransferTransition = get_json( + &fixture.control, + &ChunkKvTransferKey { + transition_id: parent.transition_id.unwrap(), + } + .to_path(), + ) + .await; + assert_eq!(transfer.phase, TransferPhase::CatalogCommitted); + tick(&fixture.control).await.unwrap(); + assert_eq!(current_catalog(&fixture.control).await.0, head); +} + +#[tokio::test] +async fn transfer_marker_waits_for_independent_recovery() { + let fixture = TestMaterialization::transferred_parent().await; + tick(&fixture.control).await.unwrap(); + let (head, page) = current_catalog(&fixture.control).await; + assert_eq!(head, fixture.head); + assert_eq!(page, fixture.page); +} + +#[tokio::test] +async fn obsolete_owner_observation_cannot_release_a_transfer_marker() { + let fixture = TestMaterialization::transferred_parent().await; + let mut obsolete = fixture.page.entries[0].clone(); + obsolete.owner_epoch += 1; + observe_partition(&fixture.control, &obsolete, true).await; + tick(&fixture.control).await.unwrap(); + let (head, page) = current_catalog(&fixture.control).await; + assert_eq!(head, fixture.head); + assert_eq!(page, fixture.page); +} + +#[tokio::test] +async fn transfer_marker_release_requires_the_exact_committed_assignment() { + for mismatch in 0..4 { + let mut fixture = TestMaterialization::transferred_parent().await; + let parent = &mut fixture.page.entries[0]; + match mismatch { + 0 => parent.owner_epoch += 1, + 1 => parent.artifact.tree_id += 1, + 2 => parent.artifact.stream_name.low += 1, + 3 => parent.transition_id = Some(Id128 { high: 999, low: 1 }), + _ => unreachable!(), + } + fixture.publish_page().await; + observe_partition(&fixture.control, &fixture.page.entries[0], true).await; + tick(&fixture.control).await.unwrap(); + let (head, page) = current_catalog(&fixture.control).await; + assert_eq!(head, fixture.head, "mismatch {mismatch}"); + assert_eq!(page, fixture.page, "mismatch {mismatch}"); + } +} diff --git a/app/crowdb-kv-server/tests/diskdb_placement_test.rs b/app/crowdb-kv-server/tests/diskdb_placement_test.rs index 6eddb6ece..495cc9ba3 100644 --- a/app/crowdb-kv-server/tests/diskdb_placement_test.rs +++ b/app/crowdb-kv-server/tests/diskdb_placement_test.rs @@ -160,6 +160,54 @@ async fn assigns_balances_and_replaces_dead_owners_without_changing_bindings() { .is_none()); } +#[tokio::test] +async fn renews_live_owner_leases_without_replacing_the_process_epoch_or_rewriting_fresh_leases() { + for remaining in [0, 1000] { + let (_, store) = registry_with_group_zero(PxLocalReplicaRole::Leader); + let control = Group0ControlPlane::acquire(&store).await.unwrap(); + instance(&control, 1, u64::MAX).await; + prepare_groups(&control).await; + let now = u64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(), + ) + .unwrap(); + let key = OwnerMapKey { + rack_id: 1, + node_id: 1, + disk_group_id: 1, + } + .to_path(); + put_json( + &control, + key.clone(), + &OwnerMapValue { + instance_id: 1, + process_epoch: 17, + lease_expiry_ms: if remaining == 0 { 1 } else { now + remaining }, + }, + ) + .await; + let initial = control.get(key.as_bytes()).await.unwrap().revision; + DiskdbPlacementMonitorDriver + .tick(&control, &descriptor()) + .await + .unwrap(); + let renewed: OwnerMapValue = get_json(&control, &key).await; + assert_eq!((renewed.instance_id, renewed.process_epoch), (1, 17)); + assert!(renewed.lease_expiry_ms > now + 60_000, "{renewed:?}"); + let revision = control.get(key.as_bytes()).await.unwrap().revision; + assert!(revision > initial); + DiskdbPlacementMonitorDriver + .tick(&control, &descriptor()) + .await + .unwrap(); + assert_eq!(control.get(key.as_bytes()).await.unwrap().revision, revision); + } +} + async fn prepare_groups(control: &Group0ControlPlane) { for dg in 1..=5 { put_json( diff --git a/app/crowdb-kv-server/tests/domain_monitor_test.rs b/app/crowdb-kv-server/tests/domain_monitor_test.rs index 75a0ce721..1b1307719 100644 --- a/app/crowdb-kv-server/tests/domain_monitor_test.rs +++ b/app/crowdb-kv-server/tests/domain_monitor_test.rs @@ -29,6 +29,8 @@ use crowdb_protocol::key::{ DomainMonitorKey, InstanceKey, ServingGrantKey, TextKey, }; +#[path = "common/grant_isolation.rs"] +mod grant_isolation; #[path = "common/grant_process.rs"] mod grant_process; #[path = "common/process_lease.rs"] @@ -38,6 +40,8 @@ mod test_process; mod catalog_publication; #[path = "common/endpoint_publication.rs"] mod endpoint_publication; +#[path = "common/split_materialization.rs"] +mod split_materialization; struct CountingDriver { ticks: Arc, @@ -544,6 +548,7 @@ async fn chunk_kv_driver_plans_dead_owner_recovery_once_and_waits_for_target() { .generation, 1 ); + grant_process::assert_returned_source_is_excluded(&control, &policy, partition_id).await; } #[allow(clippy::too_many_lines)] diff --git a/app/crowdb-kv-server/tests/startup_test.rs b/app/crowdb-kv-server/tests/startup_test.rs index 133d4df8e..170380d1a 100644 --- a/app/crowdb-kv-server/tests/startup_test.rs +++ b/app/crowdb-kv-server/tests/startup_test.rs @@ -15,6 +15,9 @@ use crowdb_kv::wal::replay::replay_group; use crowdb_kv::wal::{IoBackend, WalEngine}; use crowdb_kv_server::recovery::startup::{create_group_with_wal, store_wal_root}; +#[path = "startup_test/reclamation.rs"] +mod reclamation; + fn encode_put_payload(key: &[u8], value: &[u8]) -> Vec { let mut buf = Vec::new(); buf.extend_from_slice(&1u16.to_le_bytes()); // 1 op (u16 LE) diff --git a/app/crowdb-kv-server/tests/startup_test/reclamation.rs b/app/crowdb-kv-server/tests/startup_test/reclamation.rs new file mode 100644 index 000000000..b999207c7 --- /dev/null +++ b/app/crowdb-kv-server/tests/startup_test/reclamation.rs @@ -0,0 +1,95 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; +use crowdb_kv::wal::gc::run_gc_pass; + +#[tokio::test] +async fn restored_wal_keeps_old_segments_visible_to_safe_reclamation() { + let temp = crowdb_test_harness::test_dirs::tempdir_in_test_data("startup-wal-reclamation"); + let config = CrowDBConfig { + wal_root: temp.path().join("wal"), + config_root: temp.path().join("conf"), + data_root: temp.path().join("data"), + ..CrowDBConfig::for_tests() + }; + let backend = Arc::new(IoBackend::File); + let mut wal_config = WalConfig::with_root(store_wal_root(&config.wal_root, 0)); + wal_config.wal_segment_size = 256; + let wal = WalEngine::create(backend.clone(), wal_config, 1).await.unwrap(); + for slot in 1..=20 { + let entry = PxLogEntry { + slot, + ballot: PxBallot::new(3, 1), + term: 3, + payload: Bytes::from(encode_put_payload(b"key", &slot.to_le_bytes())), + }; + wal.append(&WALRecord::from_accepted(1, &entry)).await.unwrap(); + } + wal.seal_all().await.unwrap(); + drop(wal); + let group = create_group_with_wal( + 0, + 1, + 1, + PxLocalReplicaRole::Leader, + &config, + backend.clone(), + CrowdbTreeBackend::File, + ) + .await + .unwrap(); + let restored_wal = group.local_replica().wal().unwrap(); + assert_eq!( + restored_wal.index().slot_count(), + 20, + "startup discarded replayed slot locations" + ); + assert!(restored_wal.index().segments_snapshot().len() > 2); + assert_eq!( + run_gc_pass(restored_wal, 20).await.unwrap(), + 0, + "no durable snapshot yet" + ); + group.local_replica().learner.engine().flush(); + assert_eq!(group.local_replica().learner.engine().persist_snapshot(), 20); + drop(group); + let restarted = create_group_with_wal( + 0, + 1, + 1, + PxLocalReplicaRole::Leader, + &config, + backend.clone(), + CrowdbTreeBackend::File, + ) + .await + .unwrap(); + let restarted_wal = restarted.local_replica().wal().unwrap(); + assert_eq!( + restarted_wal.snapshot_slot(), + 20, + "restored durable floor must be immediately available" + ); + assert_eq!( + run_gc_pass(restarted_wal, 1).await.unwrap(), + 0, + "lagging learner still blocks GC" + ); + assert!( + run_gc_pass(restarted_wal, 20).await.unwrap() > 0, + "safe old segments must be reclaimable after restart" + ); + let replay = replay_group(&backend, &[store_wal_root(&config.wal_root, 0)], 1) + .await + .unwrap(); + assert!( + replay.records.len() < 20, + "reclaimed history must not return on the next restart" + ); + assert_eq!(replay.current_term, 3); + assert_eq!( + restarted.local_replica().learner.engine_get(b"key").await, + Some((20, 20_u64.to_le_bytes().to_vec())) + ); +} diff --git a/app/crowdb-web/src/diskdb.rs b/app/crowdb-web/src/diskdb.rs index 48e42c901..598021a78 100644 --- a/app/crowdb-web/src/diskdb.rs +++ b/app/crowdb-web/src/diskdb.rs @@ -7,6 +7,8 @@ pub mod lifecycle; pub mod proxy; mod remote; pub(crate) use remote::configuration as planned_configuration; +#[cfg(feature = "test-util")] +pub use remote::configuration as configuration_for_tests; pub use lifecycle::*; pub use proxy::*; diff --git a/app/crowdb-web/src/diskdb/lifecycle.rs b/app/crowdb-web/src/diskdb/lifecycle.rs index 02a135b6c..09c77c58c 100644 --- a/app/crowdb-web/src/diskdb/lifecycle.rs +++ b/app/crowdb-web/src/diskdb/lifecycle.rs @@ -26,6 +26,9 @@ pub struct DeployDiskdbBody { /// Internal HTTP health port. When omitted, derived from `rpc_port`. #[serde(default)] pub http_port: Option, + /// Explicit RPC listener for policies with independent listener ranges. + #[serde(default)] + pub rpc_listen_port: Option, /// Normal `DiskDB` heartbeat cadence, retained across service restart. #[serde(default)] pub keepalive_interval_secs: Option, @@ -36,9 +39,11 @@ pub(super) fn validate_diskdb_ports( ) -> Result<(u16, u16, u16), (StatusCode, Json)> { let listen_port = body.listen_port.unwrap_or(body.rpc_port); let http_port = body.http_port.unwrap_or_else(|| body.rpc_port.saturating_add(1)); - let rpc_listen_port = body.rpc_port.saturating_add(2); + let rpc_listen_port = body + .rpc_listen_port + .unwrap_or_else(|| body.rpc_port.saturating_add(2)); let ports = [listen_port, http_port, rpc_listen_port]; - if ports.contains(&0) || body.rpc_port > 65_533 { + if ports.contains(&0) || (body.rpc_listen_port.is_none() && body.rpc_port > 65_533) { return Err(err_400("DiskDB ports must be in the range 1..=65535")); } if ports[0] == ports[1] || ports[0] == ports[2] || ports[1] == ports[2] { diff --git a/app/crowdb-web/src/diskdb/remote.rs b/app/crowdb-web/src/diskdb/remote.rs index 29cc6a10f..542aa022c 100644 --- a/app/crowdb-web/src/diskdb/remote.rs +++ b/app/crowdb-web/src/diskdb/remote.rs @@ -7,7 +7,6 @@ use crate::{ state::AppState, }; use axum::{http::StatusCode, Json}; -use crowdb_console_shared::deployment::services; use crowdb_protocol::mgmt::node::{NodeServiceAction, NodeServiceIntent}; use std::fmt::Write as _; @@ -37,7 +36,7 @@ pub(super) async fn deploy( configuration, environment: std::collections::BTreeMap::default(), }; - let reply = services::execute(state.kv_client().await.as_ref(), &node, intent) + let reply = crate::services::remote::execute(state, &node, intent) .await .map_err(map_config_err)?; crate::services::remote::refresh(state) @@ -57,7 +56,11 @@ pub(super) async fn deploy( )) } -pub(crate) fn configuration( +/// Build a retained `DiskDB` configuration from validated listener choices. +/// +/// # Errors +/// Rejects zero, overflowing or overlapping listener ports. +pub fn configuration( node: &crowdb_console_shared::config::NodeEntry, seeds: &[String], body: &DeployDiskdbBody, diff --git a/app/crowdb-web/src/lib.rs b/app/crowdb-web/src/lib.rs index 47522a48b..510e09246 100644 --- a/app/crowdb-web/src/lib.rs +++ b/app/crowdb-web/src/lib.rs @@ -35,8 +35,20 @@ pub mod state; pub use node::observation::{start_node_observer, NodeObserver}; #[cfg(feature = "test-util")] pub use services::observation::remote_access_health_for_tests; +#[cfg(feature = "test-util")] +pub use services::observation::{ + invalidate_service_observation_for_tests, ServiceObservationMutationForTests, +}; +pub use services::observation::{start_service_observer, ServiceObserver}; +#[cfg(feature = "test-util")] +pub use services::observation::{ + start_service_observer_for_tests, ProcessObservationForTests, ServiceObservationSourceForTests, +}; pub use state::AppState; +#[cfg(feature = "test-util")] +pub use node::authority::projection::project_confirmed_servers_for_tests; + #[cfg(feature = "test-util")] pub use node::observation::{node_observation_time_for_tests, start_node_observer_for_tests}; diff --git a/app/crowdb-web/src/main.rs b/app/crowdb-web/src/main.rs index 781346ffd..89e4eab1a 100644 --- a/app/crowdb-web/src/main.rs +++ b/app/crowdb-web/src/main.rs @@ -160,6 +160,7 @@ async fn main() -> Result<(), Box> { let listener = tokio::net::TcpListener::bind(addr).await?; let _node_observer = crowdb_web::start_node_observer(&state)?; + let _service_observer = crowdb_web::start_service_observer(&state)?; let auto_join = crowdb_web::start_node_auto_join(state.clone()); if args.test_mode { let cleanup_state = state.clone(); diff --git a/app/crowdb-web/src/node.rs b/app/crowdb-web/src/node.rs index 8772d5669..f491fd2cb 100644 --- a/app/crowdb-web/src/node.rs +++ b/app/crowdb-web/src/node.rs @@ -9,7 +9,7 @@ pub(crate) mod preflight; pub(crate) mod setup; mod update; pub(crate) use update::update; -mod authority; +pub(crate) mod authority; pub(crate) mod cluster_status; mod hardware; pub(crate) mod initialization; diff --git a/app/crowdb-web/src/node/admission.rs b/app/crowdb-web/src/node/admission.rs index a1b79985a..69e67d691 100644 --- a/app/crowdb-web/src/node/admission.rs +++ b/app/crowdb-web/src/node/admission.rs @@ -179,6 +179,10 @@ async fn admit_observed( .await .map_err(map_config_err)?; record.pin_host_key(&identity.host_key).map_err(map_config_err)?; + if identity.web_port == 0 { + return Err(err_409("Authenticated target has an invalid Web listener")); + } + record.web_port = identity.web_port; if let Some(binding) = &binding { record = registry::admit( state.kv_client().await.as_ref(), @@ -189,14 +193,14 @@ async fn admit_observed( .map_err(map_config_err)?; } save(&path, &record)?; - super::setup::accept(state, &record, &target, body.setup, body.expected_formation).await?; + let setup = super::setup::accept(state, &record, &target, body.setup, body.expected_formation).await?; target.ssh_password = None; target.ssh_key = None; target.ssh_credential_ref = Some("id_ed25519".into()); if let Some(binding) = &binding { confirm_admission(state, binding, &key, &target, &mut record).await?; } - update_config(state, &record, target)?; + update_config(state, &record, target, Some(&setup))?; save(&path, &record)?; state.persist().map_err(map_persist_err)?; super::preflight::finish(state, &record)?; @@ -323,7 +327,12 @@ pub(crate) fn save(path: &Path, record: &impl serde::Serialize) -> Result<(), Fa .map_err(|error| err_502(error.to_string())) } -pub(super) fn update_config(state: &AppState, record: &NodeRecord, target: NodeEntry) -> Result<(), Failure> { +pub(super) fn update_config( + state: &AppState, + record: &NodeRecord, + target: NodeEntry, + setup: Option<&crowdb_protocol::mgmt::node::setup::NodeSetupChoices>, +) -> Result<(), Failure> { { let mut config = state.config.write().map_err(|error| err_502(error.to_string()))?; if let Some(node) = config.nodes.iter_mut().find(|node| node.id == record.node_id) { @@ -332,12 +341,15 @@ pub(super) fn update_config(state: &AppState, record: &NodeRecord, target: NodeE config.nodes.push(target); } if config.server_for_node(record.node_id).is_none() { + let (management_port, rpc_port) = setup + .and_then(crowdb_protocol::mgmt::node::setup::NodeSetupChoices::paxos_kv_ports) + .unwrap_or((10000, 10100)); let mut server = ServerEntry::new( format!("kv-{}", record.node_id), - format!("http://{}:10000", bracket_host(&record.host)), + format!("http://{}:{management_port}", bracket_host(&record.host)), ); server.node_id = Some(record.node_id); - server.rpc_url = Some(format!("{}:10100", bracket_host(&record.host))); + server.rpc_url = Some(format!("{}:{rpc_port}", bracket_host(&record.host))); config.servers.push(server); } } @@ -428,6 +440,7 @@ fn initial_record( record } else { NodeRecord { + web_port: 9090, discovery_id: id.to_string(), node_id: next_node_id(config, &records(state)?)?, physical_host_id: physical_host.to_owned(), diff --git a/app/crowdb-web/src/node/authority.rs b/app/crowdb-web/src/node/authority.rs index 28016e27f..c562d3a86 100644 --- a/app/crowdb-web/src/node/authority.rs +++ b/app/crowdb-web/src/node/authority.rs @@ -16,6 +16,8 @@ use crate::{ error::{err_409, err_502, map_config_err, ErrorBody}, state::AppState, }; +pub(crate) mod projection; + type Failure = (StatusCode, Json); pub(crate) async fn status(State(state): State) -> Json { @@ -291,36 +293,11 @@ async fn confirmed_inner( return Err(err_409("Topology publication is incomplete")); }; let mut config = state.config.write().map_err(|error| err_502(error.to_string()))?; - if config.servers.is_empty() { - config.servers = operation.intent.to_config().servers; - } - for node in registry - .nodes - .iter() - .filter(|node| node.confirmed && !node.cancelled) - { - let host = if node.host.contains(':') { - format!("[{}]", node.host) - } else { - node.host.clone() - }; - if let Some(server) = config - .servers - .iter_mut() - .find(|server| server.node_id == Some(node.node_id)) - { - server.url = format!("http://{host}:10000"); - server.rpc_url = Some(format!("{host}:10100")); - } else { - let mut server = crowdb_console_shared::config::ServerEntry::new( - format!("kv-{}", node.node_id), - format!("http://{host}:10000"), - ); - server.node_id = Some(node.node_id); - server.rpc_url = Some(format!("{host}:10100")); - config.servers.push(server); - } - } + projection::project( + &mut config.servers, + operation.intent.to_config().servers, + ®istry.nodes, + ); super::admission::save(&state.runtime_root.join("confirmed-nodes.json"), ®istry.nodes)?; Ok(()) } diff --git a/app/crowdb-web/src/node/authority/projection.rs b/app/crowdb-web/src/node/authority/projection.rs new file mode 100644 index 000000000..63d576306 --- /dev/null +++ b/app/crowdb-web/src/node/authority/projection.rs @@ -0,0 +1,61 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Project confirmed node addresses without replacing registered service identities. + +use crowdb_console_shared::{ + config::{ServerEntry, ServiceType}, + deployment::registry::NodeRecord, +}; + +pub(crate) fn project(servers: &mut Vec, bootstrap: Vec, nodes: &[NodeRecord]) { + for server in bootstrap { + if server.service_type == ServiceType::PaxosKv + && !servers.iter().any(|existing| { + existing.service_type == ServiceType::PaxosKv && existing.node_id == server.node_id + }) + { + servers.push(server); + } + } + for node in nodes.iter().filter(|node| node.confirmed && !node.cancelled) { + let host = if node.host.contains(':') { + format!("[{}]", node.host) + } else { + node.host.clone() + }; + if let Some(server) = servers.iter_mut().find(|server| { + server.node_id == Some(node.node_id) && server.service_type == ServiceType::PaxosKv + }) { + let management_port = reqwest::Url::parse(&server.url) + .ok() + .and_then(|url| url.port_or_known_default()) + .unwrap_or(10000); + let rpc_port = server + .rpc_url + .as_deref() + .and_then(|url| reqwest::Url::parse(&format!("http://{url}")).ok()) + .and_then(|url| url.port_or_known_default()) + .unwrap_or(10100); + server.url = format!("http://{host}:{management_port}"); + server.rpc_url = Some(format!("{host}:{rpc_port}")); + } else { + let mut server = crowdb_console_shared::config::ServerEntry::new( + format!("kv-{}", node.node_id), + format!("http://{host}:10000"), + ); + server.node_id = Some(node.node_id); + server.rpc_url = Some(format!("{host}:10100")); + servers.push(server); + } + } +} + +#[cfg(feature = "test-util")] +pub fn project_confirmed_servers_for_tests( + servers: &mut Vec, + bootstrap: Vec, + nodes: &[NodeRecord], +) { + project(servers, bootstrap, nodes); +} diff --git a/app/crowdb-web/src/node/auto_join/formation.rs b/app/crowdb-web/src/node/auto_join/formation.rs index 218a24fe0..1482c3b49 100644 --- a/app/crowdb-web/src/node/auto_join/formation.rs +++ b/app/crowdb-web/src/node/auto_join/formation.rs @@ -56,7 +56,7 @@ pub(super) fn candidates<'a>( "Too many fresh nodes in the cluster scope; select an unambiguous formation", )); } - if policy.initial_voters == 3 + if policy.initial_voters > 1 && snapshot.candidates.iter().any(|peer| { peer.state == CandidateState::ForeignCluster && peer.advertisement.formation.as_ref() == Some(&expected) diff --git a/app/crowdb-web/src/node/preflight.rs b/app/crowdb-web/src/node/preflight.rs index 32a7269fc..e7adcb8bb 100644 --- a/app/crowdb-web/src/node/preflight.rs +++ b/app/crowdb-web/src/node/preflight.rs @@ -113,6 +113,7 @@ pub(crate) async fn prepare( record.clone() } else { NodeRecord { + web_port: 9090, discovery_id: id.to_string(), node_id: super::admission::next_node_id(&config, &records)?, physical_host_id: handshake.physical_host_id, diff --git a/app/crowdb-web/src/node/setup.rs b/app/crowdb-web/src/node/setup.rs index 919590ef4..23432cc8b 100644 --- a/app/crowdb-web/src/node/setup.rs +++ b/app/crowdb-web/src/node/setup.rs @@ -28,7 +28,7 @@ pub(super) async fn accept( target: &NodeEntry, choices: Option, expected: Option, -) -> Result<(), Failure> { +) -> Result { let mut target = target.clone(); target.ssh_password = None; target.ssh_key = Some(super::node_key_path(state).to_string_lossy().into_owned()); @@ -75,6 +75,7 @@ pub(super) async fn accept( }) .unwrap_or_default() }; + let accepted_choices = choices.clone(); let request = NodeSetupRequest { version: 1, discovery_id: record.discovery_id.clone(), @@ -88,7 +89,7 @@ pub(super) async fn accept( let observed: NodeSetupStatus = serde_json::from_value(value).map_err(|error| err_502(error.to_string()))?; validate(state, record, &observed)?; - Ok(()) + Ok(accepted_choices) } pub(crate) async fn status( diff --git a/app/crowdb-web/src/services/deployment/planned.rs b/app/crowdb-web/src/services/deployment/planned.rs index c47679e1e..0dfe87e3b 100644 --- a/app/crowdb-web/src/services/deployment/planned.rs +++ b/app/crowdb-web/src/services/deployment/planned.rs @@ -148,6 +148,9 @@ async fn build( let name = service_name(selected.kind); let id = format!("{name}-{}", node.id); let (configuration, environment) = if selected.kind == SetupServiceKind::Diskdb { + if selected.listeners.listen_port.is_some() { + choices["rpc_listen_port"] = json!(selected.listeners.rpc_port); + } let body: crate::diskdb::DeployDiskdbBody = serde_json::from_value(choices).map_err(|error| err_400(error.to_string()))?; ( diff --git a/app/crowdb-web/src/services/deployment/remote.rs b/app/crowdb-web/src/services/deployment/remote.rs index cf0d6d7e6..5c4544a70 100644 --- a/app/crowdb-web/src/services/deployment/remote.rs +++ b/app/crowdb-web/src/services/deployment/remote.rs @@ -62,13 +62,7 @@ pub(super) async fn deploy( configuration, environment: owned_environment(state, body, &node)?, }; - let result = match crowdb_console_shared::deployment::services::execute( - state.kv_client().await.as_ref(), - &node, - intent, - ) - .await - { + let result = match crate::services::remote::execute(state, &node, intent).await { Ok(result) => result, Err(error) if error diff --git a/app/crowdb-web/src/services/observation.rs b/app/crowdb-web/src/services/observation.rs index aa0f2fea2..f3fe88a34 100644 --- a/app/crowdb-web/src/services/observation.rs +++ b/app/crowdb-web/src/services/observation.rs @@ -2,8 +2,25 @@ // Licensed under the Apache License, Version 2.0. mod access; +mod cache; +mod mutation; +mod sampling; +mod source; +#[cfg(feature = "test-util")] +mod test_support; +mod worker; #[cfg(feature = "test-util")] pub use access::remote_access_health_for_tests; +pub(crate) use cache::ServiceObservations; +#[cfg(feature = "test-util")] +pub use source::ProcessObservation as ProcessObservationForTests; +#[cfg(feature = "test-util")] +pub use test_support::{invalidate_service_observation_for_tests, ServiceObservationMutationForTests}; +pub use worker::{start_service_observer, ServiceObserver}; +#[cfg(feature = "test-util")] +pub use worker::{ + start_service_observer_for_tests, ServiceObservationSource as ServiceObservationSourceForTests, +}; use crate::state::AppState; use axum::{extract::State, Json}; @@ -31,6 +48,9 @@ pub struct ServerSummary { /// Live pid if the console currently tracks the process. #[serde(skip_serializing_if = "Option::is_none")] pub pid: Option, + /// Fresh accepted monitor evidence; absent for the legacy local inventory. + #[serde(skip_serializing_if = "Option::is_none")] + pub process_observed: Option, /// Accepted start awaits an owned PID observation and has not exhausted its launch budget. pub start_pending: bool, /// Node process namespace for a remote PID; never a Web-host PID. @@ -49,59 +69,50 @@ pub struct ServerSummary { /// # Panics /// Panics if the `RwLock` is poisoned. pub async fn http_list_servers(State(state): State) -> Json> { - let intents = if state.node_monitor_url.is_some() - && crate::node::cluster_binding(&state).ok().flatten().is_some() - { - tokio::time::timeout( - std::time::Duration::from_secs(3), - super::remote::refresh_with_intents(&state), - ) - .await - .ok() - .and_then(Result::ok) - .unwrap_or_default() - } else { - Vec::new() - }; - let snap = state.monitor_cache.snapshot().await; let cfg = state.config.read().unwrap().clone(); let remote = state.node_monitor_url.is_some(); - let remote_pids = if remote { - super::remote::processes(&state, &cfg, &intents).await + let binding = remote + .then(|| crate::node::cluster_binding(&state).ok().flatten()) + .flatten(); + let observed = state.service_observations.latest.load_full(); + // Remote inventory readers never perform authority, SSH, or endpoint I/O. + let snap = if remote { + std::collections::BTreeMap::default() } else { - super::remote::ProcessObservations::default() + state.monitor_cache.snapshot().await }; let rows = futures::future::join_all(cfg.servers.iter().map(|s| { let state = &state; let cfg = &cfg; let snap = &snap; - let remote_pids = &remote_pids; - let intents = &intents; + let observed = &observed; + let binding = &binding; async move { - let runtime_pid = match s.node_id { - Some(node_id) if s.service_type == ServiceType::Diskdb => { - state.diskdb_runtime_pid(node_id.to_string()) - } - Some(node_id) if s.service_type == ServiceType::PaxosKv => { - state.runtime_pid(node_id.to_string()) - } - _ => None, - }; - let pid = if remote { - let process_id = if s.service_type == ServiceType::PaxosKv { - "kv" - } else { - &s.id - }; - s.node_id - .and_then(|node| remote_pids.pids.get(&(node, process_id.to_owned())).copied()) + let (pid, health, start_pending, process_observed) = if remote { + let (pid, health, pending, observed) = + observed.read(binding.as_ref(), s.node_id.and_then(|id| cfg.node(id)), s); + (pid, health, pending, Some(observed)) } else { - runtime_pid.or_else(|| { + let runtime_pid = match s.node_id { + Some(node_id) if s.service_type == ServiceType::Diskdb => { + state.diskdb_runtime_pid(node_id.to_string()) + } + Some(node_id) if s.service_type == ServiceType::PaxosKv => { + state.runtime_pid(node_id.to_string()) + } + _ => None, + }; + let pid = runtime_pid.or_else(|| { s.pid .filter(|pid| crowdb_console_shared::lifecycle::process_is_alive(*pid)) - }) + }); + ( + pid, + observe_health(state, cfg, s, pid, snap, &[]).await, + false, + None, + ) }; - let health = observe_health(state, cfg, s, pid, snap, intents).await; ServerSummary { id: s.id.clone(), node_id: s.node_id, @@ -111,7 +122,8 @@ pub async fn http_list_servers(State(state): State) -> Json +// Licensed under the Apache License, Version 2.0. + +//! Identity-fenced facts sampled independently of inventory readers. + +use arc_swap::ArcSwap; +use crowdb_console_shared::{ + cluster::NodeHealth, + config::{NodeEntry, ServerEntry, ServiceType}, +}; +use crowdb_protocol::mgmt::node::{NodeBinding, NodeServiceAction, NodeServiceIntent}; +use std::{ + collections::BTreeMap, + sync::{atomic::AtomicBool, Arc}, + time::Duration, +}; +use tokio::{sync::Notify, time::Instant}; + +pub(super) const MAX_AGE: Duration = Duration::from_secs(3); +pub(super) type ServiceKey = (u64, String); + +#[derive(Default)] +pub(crate) struct ServiceObservations { + pub(crate) latest: ArcSwap, + pub(super) changed: Notify, + pub(super) running: AtomicBool, +} + +#[derive(Clone, Default)] +pub(crate) struct Inventory { + pub binding: Option, + pub authority_epoch: u64, + pub authority: Option, + pub(super) epochs: BTreeMap, + pub(super) processes: BTreeMap, + pub(super) health: BTreeMap, +} + +#[derive(Clone)] +pub(crate) struct Authority { + pub started: Instant, + pub intents: Vec, +} + +#[derive(Clone)] +pub(super) struct Sample { + pub binding: NodeBinding, + pub node: NodeEntry, + pub entry: ServerEntry, + pub intent: Option, + pub epoch: u64, +} + +#[derive(Clone)] +pub(super) struct ProcessFact { + pub(super) sample: Sample, + pub(super) started: Instant, + pub(super) pid: Option, + pub(super) exhausted: bool, + pub(super) observed: bool, +} + +#[derive(Clone)] +pub(super) struct HealthFact { + sample: Sample, + started: Instant, + pub(super) health: NodeHealth, +} + +impl Inventory { + pub fn intent(&self, entry: &ServerEntry) -> Option<&NodeServiceIntent> { + self.authority.as_ref()?.intents.iter().find(|intent| { + Some(intent.node_id) == entry.node_id + && (intent.service_id == entry.id + || (entry.service_type == ServiceType::PaxosKv && intent.kind == "kv")) + }) + } + + pub(super) fn sample(&self, binding: &NodeBinding, node: &NodeEntry, entry: &ServerEntry) -> Sample { + Sample { + binding: binding.clone(), + node: node.clone(), + entry: entry.clone(), + intent: self.intent(entry).cloned(), + epoch: self.epoch(node.id, &entry.id), + } + } + + pub(super) fn epoch(&self, node: u64, service: &str) -> u64 { + self.epochs.get(&(node, service.to_owned())).copied().unwrap_or(0) + } + + fn accepts(&self, sample: &Sample) -> bool { + self.binding.as_ref() == Some(&sample.binding) + && self.epoch(sample.node.id, &sample.entry.id) == sample.epoch + && self.intent(&sample.entry) == sample.intent.as_ref() + } + + pub fn read( + &self, + binding: Option<&NodeBinding>, + node: Option<&NodeEntry>, + entry: &ServerEntry, + ) -> (Option, NodeHealth, bool, bool) { + let Some(node) = node.filter(|_| self.binding.as_ref() == binding && binding.is_some()) else { + return (None, NodeHealth::Unknown, false, false); + }; + let key = (node.id, entry.id.clone()); + let valid = |sample: &Sample| self.accepts(sample) && sample.node == *node && sample.entry == *entry; + let process = self + .processes + .get(&key) + .filter(|fact| valid(&fact.sample) && fact.started.elapsed() <= MAX_AGE); + let pid = process.and_then(|fact| fact.pid); + let fresh_intent = self + .authority + .as_ref() + .is_some_and(|authority| authority.started.elapsed() <= MAX_AGE); + let health = self + .health + .get(&key) + .filter(|fact| valid(&fact.sample) && fact.started.elapsed() <= MAX_AGE) + .filter(|_| entry.service_type != ServiceType::AccessServer || fresh_intent) + .map_or(NodeHealth::Unknown, |fact| fact.health); + let pending = fresh_intent + && pid.is_none() + && self.intent(entry).is_some_and(|intent| { + matches!( + intent.action, + NodeServiceAction::Start | NodeServiceAction::Restart + ) + }) + && !process.is_some_and(|fact| fact.exhausted); + (pid, health, pending, process.is_some_and(|fact| fact.observed)) + } +} + +impl ServiceObservations { + pub fn synchronize(&self, binding: Option<&NodeBinding>) { + self.latest.rcu(|current| { + if current.binding.as_ref() == binding { + return Arc::clone(current); + } + Arc::new(Inventory { + binding: binding.cloned(), + authority_epoch: current.authority_epoch + 1, + ..Inventory::default() + }) + }); + } + + pub fn authority( + &self, + binding: &NodeBinding, + epoch: u64, + started: Instant, + intents: &[NodeServiceIntent], + ) -> bool { + let mut accepted = false; + let mut changed = false; + self.latest.rcu(|current| { + accepted = current.binding.as_ref() == Some(binding) + && current.authority_epoch == epoch + && current + .authority + .as_ref() + .map_or(true, |authority| started >= authority.started); + if !accepted { + changed = false; + return Arc::clone(current); + } + changed = current + .authority + .as_ref() + .map_or(true, |authority| authority.intents != intents); + let mut next = (**current).clone(); + next.authority = Some(Authority { + started, + intents: intents.to_vec(), + }); + Arc::new(next) + }); + if changed { + self.changed.notify_waiters(); + } + accepted + } + + pub(super) fn process( + &self, + sample: &Sample, + started: Instant, + pid: Option, + exhausted: bool, + observed: bool, + ) { + self.latest.rcu(|current| { + if !current.accepts(sample) { + return Arc::clone(current); + } + let mut next = (**current).clone(); + next.processes.insert( + (sample.node.id, sample.entry.id.clone()), + ProcessFact { + sample: sample.clone(), + started, + pid, + exhausted, + observed, + }, + ); + Arc::new(next) + }); + } + + pub(super) fn health(&self, sample: &Sample, started: Instant, health: NodeHealth) { + self.latest.rcu(|current| { + if !current.accepts(sample) { + return Arc::clone(current); + } + let mut next = (**current).clone(); + next.health.insert( + (sample.node.id, sample.entry.id.clone()), + HealthFact { + sample: sample.clone(), + started, + health, + }, + ); + Arc::new(next) + }); + } +} diff --git a/app/crowdb-web/src/services/observation/mutation.rs b/app/crowdb-web/src/services/observation/mutation.rs new file mode 100644 index 000000000..e192078bb --- /dev/null +++ b/app/crowdb-web/src/services/observation/mutation.rs @@ -0,0 +1,171 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Fence observations across accepted, failed, and cancelled lifecycle work. + +use super::cache::{Authority, ProcessFact, ServiceKey, ServiceObservations}; +use crate::state::AppState; +use crowdb_console_shared::config::ServiceType; +use crowdb_protocol::mgmt::node::{NodeBinding, NodeServiceIntent}; +use std::sync::Arc; +use tokio::time::Instant; + +impl ServiceObservations { + pub fn mutation(state: &AppState, node: u64, service: &str) -> Mutation { + let requested = service.to_owned(); + let config = state.config.read().unwrap(); + let service = config + .servers + .iter() + .find(|entry| { + entry.node_id == Some(node) + && (entry.id == service + || (entry.service_type == ServiceType::PaxosKv && service == format!("kv-{node}"))) + }) + .map_or_else(|| service.to_owned(), |entry| entry.id.clone()); + drop(config); + let key = (node, service); + let binding = crate::node::cluster_binding(state).ok().flatten(); + state.service_observations.synchronize(binding.as_ref()); + let mut epoch = 0; + let mut previous_operation = None; + state.service_observations.latest.rcu(|current| { + previous_operation = current.authority.as_ref().and_then(|authority| { + authority + .intents + .iter() + .find(|intent| intent.node_id == node && intent.service_id == requested) + .map(|intent| intent.operation_id.clone()) + }); + let mut next = (**current).clone(); + *next.epochs.entry(key.clone()).or_default() += 1; + epoch = next.epoch(node, &key.1); + next.authority_epoch += 1; + next.processes.remove(&key); + next.health.remove(&key); + Arc::new(next) + }); + Mutation { + observations: Arc::clone(&state.service_observations), + key, + epoch, + binding, + previous_operation, + started: Instant::now(), + } + } +} + +/// Cancellation and unknown outcomes fence in-flight reads without replaying mutations. +pub(crate) struct Mutation { + observations: Arc, + key: ServiceKey, + epoch: u64, + binding: Option, + previous_operation: Option, + started: Instant, +} + +impl Mutation { + pub(crate) fn complete( + &self, + state: &AppState, + mut intent: NodeServiceIntent, + result: &serde_json::Value, + ) { + let Some(operation) = result["operation_id"].as_str() else { + return; + }; + operation.clone_into(&mut intent.operation_id); + let Some(binding) = self + .binding + .as_ref() + .filter(|binding| crate::node::cluster_binding(state).ok().flatten().as_ref() == Some(*binding)) + else { + return; + }; + let config = state.config.read().unwrap().clone(); + let Some(node) = config.node(self.key.0) else { + return; + }; + let mut accepted = false; + self.observations.latest.rcu(|current| { + accepted = false; + if current.binding.as_ref() != Some(binding) + || current.epoch(self.key.0, &self.key.1) != self.epoch + { + return Arc::clone(current); + } + let known = current.authority.as_ref().and_then(|authority| { + authority + .intents + .iter() + .find(|known| known.service_id == intent.service_id && known.node_id == intent.node_id) + }); + if known.is_some_and(|known| { + known.operation_id != intent.operation_id + && Some(&known.operation_id) != self.previous_operation.as_ref() + }) { + return Arc::clone(current); + } + let mut next = (**current).clone(); + next.authority_epoch += 1; + *next.epochs.entry(self.key.clone()).or_default() += 1; + let authority = next.authority.get_or_insert_with(|| Authority { + started: self.started, + intents: Vec::new(), + }); + authority + .intents + .retain(|known| known.service_id != intent.service_id); + authority.intents.push(intent.clone()); + let entry = crate::services::remote::project(&intent).or_else(|| { + config + .servers + .iter() + .find(|entry| entry.id == self.key.1) + .cloned() + }); + if let Some(entry) = entry { + let sample = next.sample(binding, node, &entry); + let pid = result["pid"] + .as_u64() + .and_then(|pid| u32::try_from(pid).ok()) + .filter(|pid| *pid > 0); + next.processes.insert( + self.key.clone(), + ProcessFact { + sample, + started: self.started, + pid, + exhausted: false, + observed: pid.is_some(), + }, + ); + } + next.health.remove(&self.key); + accepted = true; + Arc::new(next) + }); + if accepted { + crate::services::remote::project_intents(state, &[intent]); + } + } +} + +impl Drop for Mutation { + fn drop(&mut self) { + self.observations.latest.rcu(|current| { + if current.binding != self.binding || current.epoch(self.key.0, &self.key.1) != self.epoch { + return Arc::clone(current); + } + let mut next = (**current).clone(); + *next.epochs.entry(self.key.clone()).or_default() += 1; + next.authority_epoch += 1; + next.processes.remove(&self.key); + next.health.remove(&self.key); + Arc::new(next) + }); + self.observations.changed.notify_waiters(); + } +} diff --git a/app/crowdb-web/src/services/observation/sampling.rs b/app/crowdb-web/src/services/observation/sampling.rs new file mode 100644 index 000000000..161c750e6 --- /dev/null +++ b/app/crowdb-web/src/services/observation/sampling.rs @@ -0,0 +1,195 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Each monitor and each endpoint has its own sampling cadence. + +use super::{ + cache::{Sample, MAX_AGE}, + source::{ProcessObservation, ServiceObservationSource}, + worker::phase_error, +}; +use crate::state::AppState; +use crowdb_console_shared::{ + cluster::NodeHealth, + config::{NodeEntry, ServerEntry, ServiceType}, +}; +use crowdb_protocol::mgmt::node::NodeBinding; +use std::{sync::Arc, time::Duration}; +use tokio::time::{Instant, MissedTickBehavior}; + +pub(super) async fn sample_node( + state: AppState, + source: Arc, + binding: NodeBinding, + node: NodeEntry, +) { + let mut transport = match source.process_observer(&state, &binding, &node) { + Ok(transport) => transport, + Err(error) => { + phase_error(&state, node.id, "processes", Instant::now(), &error); + return; + } + }; + let mut interval = tokio::time::interval(Duration::from_secs(1)); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + tokio::select! { _ = interval.tick() => {}, () = state.service_observations.changed.notified() => {} } + let config = state.config.read().unwrap().clone(); + if crate::node::cluster_binding(&state).ok().flatten().as_ref() != Some(&binding) + || config.node(node.id) != Some(&node) + { + return; + } + let observed = state.service_observations.latest.load_full(); + let samples: Vec<_> = config + .servers + .iter() + .filter(|entry| entry.node_id == Some(node.id)) + .map(|entry| observed.sample(&binding, &node, entry)) + .collect(); + sample_processes(&state, source.as_ref(), &mut transport, &binding, &node, &samples).await; + } +} + +async fn sample_processes( + state: &AppState, + source: &dyn ServiceObservationSource, + observer: &mut Option>, + binding: &NodeBinding, + node: &NodeEntry, + samples: &[Sample], +) { + let started = Instant::now(); + let read = async { + match observer { + Some(observer) => observer.read().await, + None => source.processes(state, binding, node).await, + } + }; + let result = match tokio::time::timeout(MAX_AGE, read).await { + Ok(Ok(value)) => Some(value), + Ok(Err(error)) => { + phase_error(state, node.id, "processes", started, &error); + None + } + Err(_) => { + phase_error( + state, + node.id, + "processes", + started, + "three-second sampling deadline exceeded", + ); + None + } + }; + let bootstrap = serde_json::to_value(&binding.bootstrap).expect("bootstrap serialization is infallible"); + let value = result.filter(|value| { + value["node_id"] == node.id + && value["cluster_id"] == binding.bootstrap.cluster_id + && value["bootstrap"] == bootstrap + }); + if value.is_none() { + tracing::debug!( + node_id = node.id, + phase = "processes", + "no current authenticated process observation" + ); + } + tracing::debug!( + node_id = node.id, + phase = "processes", + elapsed_ms = started.elapsed().as_millis(), + "service inventory sampled" + ); + for sample in samples { + let id = if sample.entry.service_type == ServiceType::PaxosKv { + "kv" + } else { + &sample.entry.id + }; + let owned = value.as_ref().filter(|value| { + sample + .intent + .as_ref() + .map_or(sample.entry.service_type == ServiceType::PaxosKv, |intent| { + value["process_operations"][id] == intent.operation_id + }) + }); + let pid = owned + .and_then(|value| value["processes"][id].as_u64()) + .and_then(|pid| u32::try_from(pid).ok()) + .filter(|pid| *pid > 0); + let exhausted = value + .as_ref() + .zip(sample.intent.as_ref()) + .is_some_and(|(value, intent)| { + value["service_failures"][&intent.service_id]["operation_id"] == intent.operation_id + && value["service_failures"][&intent.service_id]["exhausted"] == true + }); + let has_process_evidence = value + .as_ref() + .and_then(|value| value["processes"].as_object()) + .is_some_and(|processes| match processes.get(id) { + None => sample.intent.is_some() || sample.entry.service_type == ServiceType::PaxosKv, + Some(value) => value + .as_u64() + .and_then(|pid| u32::try_from(pid).ok()) + .is_some_and(|pid| pid > 0 && owned.is_some()), + }); + state + .service_observations + .process(sample, started, pid, exhausted, has_process_evidence); + } +} + +pub(super) async fn sample_service_health( + state: AppState, + source: Arc, + binding: NodeBinding, + node: NodeEntry, + entry: ServerEntry, +) { + let mut interval = tokio::time::interval(Duration::from_secs(1)); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + tokio::select! { _ = interval.tick() => {}, () = state.service_observations.changed.notified() => {} } + let config = state.config.read().unwrap().clone(); + if crate::node::cluster_binding(&state).ok().flatten().as_ref() != Some(&binding) + || config.node(node.id) != Some(&node) + || !config.servers.contains(&entry) + { + return; + } + let observed = state.service_observations.latest.load_full(); + let sample = observed.sample(&binding, &node, &entry); + let intents = observed + .authority + .as_ref() + .filter(|authority| authority.started.elapsed() <= MAX_AGE) + .map_or(&[][..], |authority| authority.intents.as_slice()); + let started = Instant::now(); + let pid = observed + .read(Some(&sample.binding), Some(&sample.node), &sample.entry) + .0; + let response = tokio::select! { + biased; + () = state.service_observations.changed.notified() => continue, + result = tokio::time::timeout(MAX_AGE, source.health(&state, &config, &sample.entry, pid, intents)) => result, + }; + let health = if let Ok(health) = response { + health + } else { + phase_error( + &state, + sample.node.id, + "health", + started, + "three-second sampling deadline exceeded", + ); + NodeHealth::Unknown + }; + tracing::debug!(node_id = sample.node.id, service_id = %sample.entry.id, phase = "health", elapsed_ms = started.elapsed().as_millis(), ?health, "service endpoint sampled"); + state.service_observations.health(&sample, started, health); + } +} diff --git a/app/crowdb-web/src/services/observation/source.rs b/app/crowdb-web/src/services/observation/source.rs new file mode 100644 index 000000000..e28c6c6ff --- /dev/null +++ b/app/crowdb-web/src/services/observation/source.rs @@ -0,0 +1,125 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Authenticated process and real endpoint transports for inventory sampling. + +use crate::state::AppState; +use crowdb_console_shared::{ + cluster::NodeHealth, + config::{NodeEntry, ServerEntry}, + ConsoleConfig, +}; +use crowdb_protocol::mgmt::node::{NodeBinding, NodeControl, NodeServiceIntent}; +use serde_json::Value; + +/// One node sampler owns this reader throughout its bound configuration. +#[async_trait::async_trait] +pub trait ProcessObservation: Send { + /// Read one fresh monitor response; never return a retained observation. + async fn read(&mut self) -> Result; +} + +#[async_trait::async_trait] +impl ProcessObservation for crowdb_console_shared::deployment::admission::RemoteProcessObservation { + async fn read(&mut self) -> Result { + Self::read(self).await.map_err(|error| error.to_string()) + } +} + +/// Observation transports; injected only by focused regression fixtures. +#[async_trait::async_trait] +pub trait ServiceObservationSource: Send + Sync + 'static { + /// Create an owned process transport when the source supports connection reuse. + /// + /// # Errors + /// Rejects an invalid bound transport configuration. + fn process_observer( + &self, + _state: &AppState, + _binding: &NodeBinding, + _node: &NodeEntry, + ) -> Result>, String> { + Ok(None) + } + /// Read the committed service view without accepting mutations. + async fn intents(&self, state: &AppState) -> Result, String>; + /// Read authenticated, bound monitor-owned children. + async fn processes( + &self, + state: &AppState, + binding: &NodeBinding, + node: &NodeEntry, + ) -> Result; + /// Probe the service's actual endpoint, independently of process presence. + async fn health( + &self, + state: &AppState, + config: &ConsoleConfig, + entry: &ServerEntry, + pid: Option, + intents: &[NodeServiceIntent], + ) -> NodeHealth; +} + +pub(super) struct Production; +#[async_trait::async_trait] +impl ServiceObservationSource for Production { + fn process_observer( + &self, + state: &AppState, + binding: &NodeBinding, + node: &NodeEntry, + ) -> Result>, String> { + if node.id == binding.node_id { + return Ok(None); + } + let mut target = node.clone(); + target.ssh_password = None; + target.ssh_key = Some(crate::node::node_key_path(state).to_string_lossy().into_owned()); + let observer = crowdb_console_shared::deployment::admission::RemoteProcessObservation::new( + target, + binding.clone(), + ) + .map_err(|error| error.to_string())?; + Ok(Some(Box::new(observer))) + } + async fn intents(&self, state: &AppState) -> Result, String> { + crowdb_console_shared::deployment::services::list(state.kv_client().await.as_ref()) + .await + .map_err(|error| error.to_string()) + } + async fn processes( + &self, + state: &AppState, + binding: &NodeBinding, + node: &NodeEntry, + ) -> Result { + let command = NodeControl::Processes { + cluster_id: binding.bootstrap.cluster_id.clone(), + node_id: node.id, + }; + let result = if node.id == binding.node_id { + crowdb_console_shared::deployment::admission::local_control( + &crate::node::control_socket(), + &command, + ) + .await + } else { + let mut target = node.clone(); + target.ssh_key = Some(crate::node::node_key_path(state).to_string_lossy().into_owned()); + crowdb_console_shared::deployment::admission::remote_control(&target, &command).await + }; + result.map_err(|error| error.to_string()) + } + async fn health( + &self, + state: &AppState, + config: &ConsoleConfig, + entry: &ServerEntry, + pid: Option, + intents: &[NodeServiceIntent], + ) -> NodeHealth { + let snapshot = state.monitor_cache.snapshot().await; + super::observe_health(state, config, entry, pid, &snapshot, intents).await + } +} diff --git a/app/crowdb-web/src/services/observation/test_support.rs b/app/crowdb-web/src/services/observation/test_support.rs new file mode 100644 index 000000000..fd74ad7b3 --- /dev/null +++ b/app/crowdb-web/src/services/observation/test_support.rs @@ -0,0 +1,32 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{cache::ServiceObservations, mutation::Mutation}; +use crate::AppState; +use crowdb_protocol::mgmt::node::NodeServiceIntent; + +/// Holds the production observation fence while a fixture controls a mutation. +pub struct ServiceObservationMutationForTests { + state: AppState, + mutation: Mutation, +} + +/// Invalidate one target using the same fence as remote lifecycle execution. +#[must_use] +pub fn invalidate_service_observation_for_tests( + state: &AppState, + node: u64, + service: &str, +) -> ServiceObservationMutationForTests { + ServiceObservationMutationForTests { + state: state.clone(), + mutation: ServiceObservations::mutation(state, node, service), + } +} + +impl ServiceObservationMutationForTests { + /// Publish an actual monitor reply through the production completion path. + pub fn complete(&self, intent: NodeServiceIntent, reply: &serde_json::Value) { + self.mutation.complete(&self.state, intent, reply); + } +} diff --git a/app/crowdb-web/src/services/observation/worker.rs b/app/crowdb-web/src/services/observation/worker.rs new file mode 100644 index 000000000..3ec5777f2 --- /dev/null +++ b/app/crowdb-web/src/services/observation/worker.rs @@ -0,0 +1,198 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Authority and each node publish independently; reads never drive sampling. + +use super::{ + cache::MAX_AGE, + sampling::{sample_node, sample_service_health}, + source::Production, + ServiceObservations, +}; +use crate::state::AppState; +use crowdb_console_shared::config::{NodeEntry, ServerEntry}; +use crowdb_protocol::mgmt::node::NodeBinding; +use std::{ + collections::BTreeMap, + sync::{atomic::Ordering, Arc}, + time::Duration, +}; +use tokio::{ + task::{AbortHandle, JoinSet}, + time::{Instant, MissedTickBehavior}, +}; + +pub use super::source::ServiceObservationSource; + +/// Stops the coordinator and all its node samplers when the Web listener exits. +pub struct ServiceObserver(tokio::task::JoinHandle<()>); +impl Drop for ServiceObserver { + fn drop(&mut self) { + self.0.abort(); + } +} +struct SamplingLease(Arc); +impl Drop for SamplingLease { + fn drop(&mut self) { + self.0.running.store(false, Ordering::Release); + } +} + +/// Start independent service inventory observations for monitor-owned Web. +/// +/// # Errors +/// Rejects an already running observer. +pub fn start_service_observer(state: &AppState) -> Result, &'static str> { + if state.node_monitor_url.is_none() { + return Ok(None); + } + start(state, Arc::new(Production)).map(Some) +} + +/// Start the production sampler with deterministic observation transports. +/// +/// # Errors +/// Rejects an unowned Web or a duplicate observer. +#[cfg(feature = "test-util")] +pub fn start_service_observer_for_tests( + state: &AppState, + source: Arc, +) -> Result { + start(state, source) +} + +fn start( + state: &AppState, + source: Arc, +) -> Result { + if state.node_monitor_url.is_none() { + return Err("Service sampling requires an owning monitor"); + } + state + .service_observations + .running + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .map_err(|_| "Service observation sampler is already running")?; + let state = state.clone(); + let lease = SamplingLease(Arc::clone(&state.service_observations)); + Ok(ServiceObserver(tokio::spawn(async move { + let _lease = lease; + coordinate(state, source).await; + }))) +} + +async fn coordinate(state: AppState, source: Arc) { + let mut children = JoinSet::new(); + let mut authority_task = children.spawn(authority(state.clone(), Arc::clone(&source))); + let mut nodes: BTreeMap = BTreeMap::new(); + let mut health: BTreeMap<(u64, String), (NodeBinding, NodeEntry, ServerEntry, AbortHandle)> = + BTreeMap::new(); + let mut interval = tokio::time::interval(Duration::from_millis(100)); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + tokio::select! { + _ = interval.tick() => { + if authority_task.is_finished() { + authority_task = children.spawn(authority(state.clone(), Arc::clone(&source))); + } + let binding = crate::node::cluster_binding(&state).ok().flatten(); + state.service_observations.synchronize(binding.as_ref()); + let config = state.config.read().unwrap().clone(); + nodes.retain(|id, (old_binding, old_node, task)| { + let retain = !task.is_finished() && binding.as_ref() == Some(old_binding) && config.node(*id) == Some(old_node); + if !retain { task.abort(); } + retain + }); + health.retain(|(node, id), (old_binding, old_node, entry, task)| { + let retain = !task.is_finished() && binding.as_ref() == Some(old_binding) + && config.node(*node) == Some(old_node) && config.servers.iter().any(|current| current.id == *id && current == entry); + if !retain { task.abort(); } + retain + }); + if let Some(binding) = binding { + for node in &config.nodes { + if nodes.contains_key(&node.id) { continue; } + let handle = children.spawn(sample_node(state.clone(), Arc::clone(&source), binding.clone(), node.clone())); + nodes.insert(node.id, (binding.clone(), node.clone(), handle)); + } + for entry in &config.servers { + let Some(node) = entry.node_id.and_then(|id| config.node(id)) else { continue; }; + let key = (node.id, entry.id.clone()); + if health.contains_key(&key) { continue; } + let handle = children.spawn(sample_service_health(state.clone(), Arc::clone(&source), binding.clone(), node.clone(), entry.clone())); + health.insert(key, (binding.clone(), node.clone(), entry.clone(), handle)); + } + } + } + result = children.join_next(), if !children.is_empty() => { + if let Some(Err(error)) = result { + if !error.is_cancelled() { tracing::error!(%error, "service observation worker stopped; unavailable observations expire"); } + } + } + } + } +} + +async fn authority(state: AppState, source: Arc) { + let mut interval = tokio::time::interval(Duration::from_secs(1)); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + tokio::select! { _ = interval.tick() => {}, () = state.service_observations.changed.notified() => {} } + let Some(binding) = crate::node::cluster_binding(&state).ok().flatten() else { + continue; + }; + state.service_observations.synchronize(Some(&binding)); + let epoch = state.service_observations.latest.load().authority_epoch; + let started = Instant::now(); + let intents = match tokio::time::timeout(MAX_AGE, source.intents(&state)).await { + Ok(Ok(intents)) => intents, + Ok(Err(error)) => { + phase_error(&state, binding.node_id, "authority", started, &error); + continue; + } + Err(_) => { + phase_error( + &state, + binding.node_id, + "authority", + started, + "three-second sampling deadline exceeded", + ); + continue; + } + }; + tracing::debug!( + phase = "authority", + elapsed_ms = started.elapsed().as_millis(), + "service inventory sampled" + ); + if crate::node::cluster_binding(&state).ok().flatten().as_ref() != Some(&binding) { + continue; + } + let intents: Vec<_> = intents + .into_iter() + .filter(|intent| intent.cluster_id == binding.bootstrap.cluster_id) + .collect(); + if state + .service_observations + .authority(&binding, epoch, started, &intents) + { + crate::services::remote::project_intents(&state, &intents); + } + } +} + +pub(super) fn phase_error(state: &AppState, node: u64, phase: &str, started: Instant, error: &str) { + let diagnostic: String = error + .chars() + .filter(|character| !character.is_control()) + .take(256) + .collect(); + if state.should_warn( + &format!("service-observation/{node}/{phase}"), + Duration::from_secs(30), + ) { + tracing::warn!(node_id = node, phase, elapsed_ms = started.elapsed().as_millis(), error = %diagnostic, + "service observation unavailable; stale evidence expires and the next sample retries the read"); + } +} diff --git a/app/crowdb-web/src/services/remote.rs b/app/crowdb-web/src/services/remote.rs index 86fa5267a..78ab8295f 100644 --- a/app/crowdb-web/src/services/remote.rs +++ b/app/crowdb-web/src/services/remote.rs @@ -13,82 +13,9 @@ use crowdb_console_shared::{ config::{ServerEntry, ServiceType}, deployment::services, }; -use crowdb_protocol::mgmt::node::{NodeControl, NodeServiceAction, NodeServiceIntent}; +use crowdb_protocol::mgmt::node::{NodeServiceAction, NodeServiceIntent}; use serde_json::{json, Value}; -#[derive(Default)] -pub(super) struct ProcessObservations { - pub pids: std::collections::BTreeMap<(u64, String), u32>, - exhausted: std::collections::BTreeSet<(u64, String)>, -} - -impl ProcessObservations { - fn record(&mut self, node: u64, value: &Value, intents: &[NodeServiceIntent]) { - if let Some(processes) = value["processes"].as_object() { - for (id, value) in processes { - if let Some(pid) = value - .as_u64() - .and_then(|pid| u32::try_from(pid).ok()) - .filter(|pid| *pid > 0) - { - self.pids.insert((node, id.clone()), pid); - } - } - } - for intent in intents.iter().filter(|intent| intent.node_id == node) { - let failure = &value["service_failures"][&intent.service_id]; - if failure["operation_id"] == intent.operation_id && failure["exhausted"] == true { - self.exhausted.insert((node, intent.service_id.clone())); - } - } - } - - pub fn start_pending(&self, entry: &ServerEntry, remote: bool, pid: Option) -> bool { - remote - && entry.auto_start - && pid.is_none() - && !entry - .node_id - .is_some_and(|node| self.exhausted.contains(&(node, entry.id.clone()))) - } -} - -pub(super) async fn processes( - state: &AppState, - config: &crowdb_console_shared::ConsoleConfig, - intents: &[NodeServiceIntent], -) -> ProcessObservations { - let Some(binding) = crate::node::cluster_binding(state).ok().flatten() else { - return ProcessObservations::default(); - }; - let cluster = &binding.bootstrap.cluster_id; - let observations = futures::future::join_all(config.nodes.iter().map(|node| async move { - let mut target = node.clone(); - target.ssh_key = Some(crate::node::node_key_path(state).to_string_lossy().into_owned()); - let command = NodeControl::Processes { - cluster_id: cluster.clone(), - node_id: node.id, - }; - let result = tokio::time::timeout( - std::time::Duration::from_secs(3), - crowdb_console_shared::deployment::admission::remote_control(&target, &command), - ) - .await - .ok()? - .ok()?; - if result["node_id"] != node.id || result["cluster_id"] != *cluster { - return None; - } - Some((node.id, result)) - })) - .await; - let mut processes = ProcessObservations::default(); - for (node, observation) in observations.into_iter().flatten() { - processes.record(node, &observation, intents); - } - processes -} - pub(crate) async fn act( state: &AppState, id: &str, @@ -115,9 +42,7 @@ pub(crate) async fn act( node.ssh_key = Some(crate::node::node_key_path(state).to_string_lossy().into_owned()); intent.operation_id = uuid::Uuid::new_v4().to_string(); intent.action = action; - let result = services::execute(state.kv_client().await.as_ref(), &node, intent) - .await - .map_err(map_config_err)?; + let result = execute(state, &node, intent).await.map_err(map_config_err)?; refresh(state).await.map_err(map_config_err)?; Ok(Json( json!({"id": id, "pid":result["pid"], "removed":matches!(action, NodeServiceAction::Delete), "data_preserved":true}), @@ -131,12 +56,43 @@ pub(crate) async fn refresh(state: &AppState) -> crowdb_console_shared::error::R pub(super) async fn refresh_with_intents( state: &AppState, ) -> crowdb_console_shared::error::Result> { - let intents = services::list(state.kv_client().await.as_ref()).await?; - let mut config = state - .config - .write() - .map_err(|error| crowdb_console_shared::error::Error::Config(error.to_string()))?; - for intent in &intents { + let binding = crate::node::cluster_binding(state).ok().flatten(); + state.service_observations.synchronize(binding.as_ref()); + let epoch = state.service_observations.latest.load().authority_epoch; + let started = tokio::time::Instant::now(); + let intents: Vec<_> = services::list(state.kv_client().await.as_ref()) + .await? + .into_iter() + .filter(|intent| { + binding + .as_ref() + .is_some_and(|binding| intent.cluster_id == binding.bootstrap.cluster_id) + }) + .collect(); + if binding.as_ref().is_some_and(|binding| { + crate::node::cluster_binding(state).ok().flatten().as_ref() == Some(binding) + && state + .service_observations + .authority(binding, epoch, started, &intents) + }) { + project_intents(state, &intents); + } + Ok(intents) +} + +pub(super) fn project_intents(state: &AppState, intents: &[NodeServiceIntent]) { + let mut config = state.config.write().unwrap(); + let current = state.service_observations.latest.load_full(); + if crate::node::cluster_binding(state).ok().flatten() != current.binding { + return; + } + let Some(authority) = ¤t.authority else { + return; + }; + for intent in intents { + if !authority.intents.contains(intent) { + continue; + } // KV endpoints come from admitted node bootstrap, not auxiliary // service configuration. Its lifecycle intent must not erase them. if intent.kind == "kv" { @@ -150,10 +106,9 @@ pub(super) async fn refresh_with_intents( config.servers.push(entry); } } - Ok(intents) } -fn project(intent: &NodeServiceIntent) -> Option { +pub(super) fn project(intent: &NodeServiceIntent) -> Option { let value: toml::Value = toml::from_str(&intent.configuration).ok()?; let (kind, http, rpc) = match intent.kind.as_str() { "diskdb" => ( @@ -259,11 +214,16 @@ pub(crate) async fn kv_action( configuration: format!("node_id = {node_id}\n"), environment: std::collections::BTreeMap::new(), }; - services::execute(state.kv_client().await.as_ref(), &node, intent) - .await - .map_err(map_config_err) + execute(state, &node, intent).await.map_err(map_config_err) } -#[cfg(test)] -#[path = "../../tests/internal/service_start_pending_test.rs"] -mod tests; +pub(crate) async fn execute( + state: &AppState, + node: &crowdb_console_shared::config::NodeEntry, + intent: NodeServiceIntent, +) -> crowdb_console_shared::error::Result { + let mutation = super::observation::ServiceObservations::mutation(state, node.id, &intent.service_id); + let result = services::execute(state.kv_client().await.as_ref(), node, intent.clone()).await?; + mutation.complete(state, intent, &result); + Ok(result) +} diff --git a/app/crowdb-web/src/state.rs b/app/crowdb-web/src/state.rs index 4a0428b51..2120606b8 100644 --- a/app/crowdb-web/src/state.rs +++ b/app/crowdb-web/src/state.rs @@ -22,6 +22,7 @@ use crowdb_console_shared::{config::ServerEntry, ConsoleConfig}; #[derive(Clone)] pub struct AppState { pub(crate) node_observations: Arc, + pub(crate) service_observations: Arc, pub(crate) service_operations: Arc>>, pub config: Arc>, pub(crate) config_path: Option, @@ -100,6 +101,7 @@ impl AppState { pub fn with_runtime_root(config: ConsoleConfig, runtime_root: PathBuf) -> Self { Self { node_observations: Arc::default(), + service_observations: Arc::default(), service_operations: Arc::new(arc_swap::ArcSwap::from_pointee(std::collections::HashSet::new())), config: Arc::new(RwLock::new(config)), config_path: None, diff --git a/app/crowdb-web/tests/auto_formation_test.rs b/app/crowdb-web/tests/auto_formation_test.rs index 37a9e5ce3..c72a3e0f3 100644 --- a/app/crowdb-web/tests/auto_formation_test.rs +++ b/app/crowdb-web/tests/auto_formation_test.rs @@ -134,3 +134,55 @@ async fn missing_exposed_empty_or_corrupt_local_credentials_never_fall_back_to_a assert!(!fixture.root.join("accepted-node.json").exists()); } } + +#[tokio::test] +async fn five_voter_scope_preserves_incomplete_overfull_and_foreign_member_fences() { + let expected = policy(5, DeploymentMode::Local); + for count in 0..4 { + let peers = vec![(expected.formation(), CandidateState::Unbound); count]; + let fixture = TestFormationFixture::start(expected.clone(), peers, 1).await; + let report = fixture.observation().await; + assert_eq!(report["phase"], "waiting", "{report}"); + assert_eq!(report["required_voters"], 5); + assert_eq!(report["observed_voters"], count + 1); + fixture.assert_no_admission(); + } + for state in [CandidateState::ForeignCluster, CandidateState::IdentityConflict] { + let fixture = + TestFormationFixture::start(expected.clone(), vec![(expected.formation(), state)], 1).await; + let report = fixture.observation().await; + assert_eq!(report["phase"], "failed", "{report}"); + fixture.assert_no_admission(); + } + let fixture = TestFormationFixture::start( + expected.clone(), + vec![(expected.formation(), CandidateState::Unbound); 5], + 1, + ) + .await; + let report = fixture.observation().await; + assert_eq!(report["phase"], "failed", "{report}"); + assert!(report["error"].as_str().unwrap().contains("Too many fresh nodes")); + fixture.assert_no_admission(); + let fixture = TestFormationFixture::start( + expected.clone(), + vec![(expected.formation(), CandidateState::Unbound); 4], + 99, + ) + .await; + let report = fixture.observation().await; + assert_eq!(report["phase"], "waiting", "{report}"); + assert_eq!(report["coordinator"], "00000000-0000-4000-8000-000000000002"); + fixture.assert_no_admission(); + let production = policy(5, DeploymentMode::Production); + let fixture = TestFormationFixture::with_password( + production.clone(), + vec![(production.formation(), CandidateState::Unbound); 4], + 1, + Some((b"local-test-password", 0o600)), + ) + .await; + let report = fixture.observation().await; + assert_eq!(report["phase"], "needs-rack", "{report}"); + fixture.assert_no_admission(); +} diff --git a/app/crowdb-web/tests/auto_join_test.rs b/app/crowdb-web/tests/auto_join_test.rs index 6597a4d40..e780fd419 100644 --- a/app/crowdb-web/tests/auto_join_test.rs +++ b/app/crowdb-web/tests/auto_join_test.rs @@ -189,6 +189,7 @@ async fn two_authenticated_nodes_wait_in_uuid_order_without_initializing() { ssh_credential_ref: Some("id_ed25519".into()), }); let record = NodeRecord { + web_port: 9090, discovery_id: discovery_id.into(), node_id, physical_host_id: "host".into(), diff --git a/app/crowdb-web/tests/cluster_target_test.rs b/app/crowdb-web/tests/cluster_target_test.rs index 13c5e66b0..17fdcefe0 100644 --- a/app/crowdb-web/tests/cluster_target_test.rs +++ b/app/crowdb-web/tests/cluster_target_test.rs @@ -143,6 +143,7 @@ async fn target_uses_confirmed_current_addresses_without_rewriting_the_manifest( let registry = NodeRegistry { cluster_id: fixture.prepared.identity.cluster_id.clone(), nodes: vec![NodeRecord { + web_port: 9090, discovery_id: uuid::Uuid::new_v4().to_string(), node_id: 2, physical_host_id: "test-host".into(), diff --git a/app/crowdb-web/tests/common/process_observed.rs b/app/crowdb-web/tests/common/process_observed.rs new file mode 100644 index 000000000..32e2974f7 --- /dev/null +++ b/app/crowdb-web/tests/common/process_observed.rs @@ -0,0 +1,175 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::fixture::{TestServiceInventory, TestSource}; +use crowdb_console_shared::{ + cluster::NodeHealth, + config::{NodeEntry, ServerEntry, ServiceType}, + ConsoleConfig, +}; +use crowdb_protocol::mgmt::node::{NodeBinding, NodeServiceIntent}; +use crowdb_web::{AppState, ServiceObservationSourceForTests}; +use serde_json::{json, Value}; +use std::{ + sync::{ + atomic::{AtomicU32, Ordering}, + Arc, + }, + time::Duration, +}; + +pub const RUNNING: u32 = 0; +pub const ABSENT: u32 = 1; +pub const READ_ERROR: u32 = 2; +pub const FOREIGN_BOOTSTRAP: u32 = 3; +pub const FOREIGN_OPERATION: u32 = 4; +pub const EXHAUSTED: u32 = 5; +pub const FOREIGN_EXHAUSTION: u32 = 6; +pub const MISSING_MAP: u32 = 7; +pub const NULL_MAP: u32 = 8; +pub const ARRAY_MAP: u32 = 9; +pub const ZERO_PID: u32 = 10; +pub const STRING_PID: u32 = 11; +pub const FOREIGN_NODE: u32 = 12; +pub const FOREIGN_CLUSTER: u32 = 13; + +struct TestProcessStateSource { + base: Arc, + mode: Arc, + bootstrap_pkv: bool, +} + +#[async_trait::async_trait] +impl ServiceObservationSourceForTests for TestProcessStateSource { + async fn intents(&self, _: &AppState) -> Result, String> { + Ok(self + .base + .intents + .load_full() + .iter() + .filter(|intent| !(self.bootstrap_pkv && intent.node_id == 1)) + .cloned() + .collect()) + } + + async fn processes( + &self, + _: &AppState, + binding: &NodeBinding, + node: &NodeEntry, + ) -> Result { + let mode = self.mode.load(Ordering::Relaxed); + if mode == READ_ERROR { + return Err("controlled authenticated read unavailable".into()); + } + let intent = self + .base + .intents + .load_full() + .iter() + .find(|i| i.node_id == node.id) + .unwrap() + .clone(); + let key = if self.bootstrap_pkv && node.id == 1 { + "kv".to_owned() + } else { + intent.service_id.clone() + }; + let mut bootstrap = binding.bootstrap.clone(); + if mode == FOREIGN_BOOTSTRAP { + bootstrap.operation_id = "foreign-bootstrap".into(); + } + let absent = [ABSENT, EXHAUSTED, FOREIGN_EXHAUSTION].contains(&mode); + let operation = if [FOREIGN_OPERATION, FOREIGN_EXHAUSTION].contains(&mode) { + "foreign-operation" + } else { + &intent.operation_id + }; + let mut value = json!({ + "cluster_id": binding.bootstrap.cluster_id, "node_id": node.id, "bootstrap": bootstrap, + "processes": if absent { json!({}) } else { json!({key.clone(): 100 + node.id}) }, + // Production monitor exposes execution ownership only for live children. + "process_operations": if absent || (self.bootstrap_pkv && node.id == 1) { json!({}) } else { json!({key.clone(): operation}) }, + "service_failures": if [EXHAUSTED, FOREIGN_EXHAUSTION].contains(&mode) { + json!({intent.service_id: {"operation_id": operation, "exhausted": true}}) + } else { json!({}) }, + }); + match mode { + MISSING_MAP => { + value.as_object_mut().unwrap().remove("processes"); + } + NULL_MAP => value["processes"] = Value::Null, + ARRAY_MAP => value["processes"] = json!([]), + ZERO_PID => value["processes"][&key] = json!(0), + STRING_PID => value["processes"][&key] = json!("not-a-pid"), + FOREIGN_NODE => value["node_id"] = json!(99), + FOREIGN_CLUSTER => value["cluster_id"] = json!("foreign-cluster"), + _ => {} + } + Ok(value) + } + + async fn health( + &self, + _: &AppState, + _: &ConsoleConfig, + _: &ServerEntry, + _: Option, + _: &[NodeServiceIntent], + ) -> NodeHealth { + NodeHealth::Up + } +} + +pub fn start(fixture: &mut TestServiceInventory, bootstrap_pkv: bool) -> Arc { + if bootstrap_pkv { + let mut config = fixture.state.config.write().unwrap(); + let entry = config.servers.iter_mut().find(|s| s.node_id == Some(1)).unwrap(); + entry.id = "kv-1".into(); + entry.service_type = ServiceType::PaxosKv; + entry.url = "http://127.0.0.1:10000".into(); + entry.rpc_url = Some("127.0.0.1:10100".into()); + entry.rest_port = Some(10000); + entry.rpc_port = Some(10100); + } + let mode = Arc::new(AtomicU32::new(RUNNING)); + fixture.observer = Some( + crowdb_web::start_service_observer_for_tests( + &fixture.state, + Arc::new(TestProcessStateSource { + base: fixture.source.clone(), + mode: mode.clone(), + bootstrap_pkv, + }), + ) + .unwrap(), + ); + mode +} + +pub async fn wait(fixture: &TestServiceInventory, expected: &str) -> Value { + tokio::time::timeout(Duration::from_secs(2), async { + loop { + let rows = fixture.rows().await; + let row = rows + .as_array() + .unwrap() + .iter() + .find(|r| r["node_id"] == 1) + .unwrap(); + let observed = row["process_observed"].as_bool(); + let matches = match expected { + "running" => observed == Some(true) && row["pid"].as_u64().is_some() && row["health"] == "up", + "stopped" => observed == Some(true) && row["pid"].is_null(), + "unknown" => observed == Some(false) && row["pid"].is_null(), + _ => panic!("unexpected process evidence state"), + }; + if matches { + return row.clone(); + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("production observation did not distinguish validity from PID presence") +} diff --git a/app/crowdb-web/tests/common/process_transport.rs b/app/crowdb-web/tests/common/process_transport.rs new file mode 100644 index 000000000..c9d4e76ae --- /dev/null +++ b/app/crowdb-web/tests/common/process_transport.rs @@ -0,0 +1,90 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::fixture::TestSource; +use crowdb_console_shared::{ + cluster::NodeHealth, + config::{NodeEntry, ServerEntry}, + ConsoleConfig, +}; +use crowdb_protocol::mgmt::node::{NodeBinding, NodeServiceIntent}; +use crowdb_web::{AppState, ProcessObservationForTests, ServiceObservationSourceForTests}; +use serde_json::Value; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +#[derive(Default)] +pub struct TestCounts { + pub created: [AtomicUsize; 3], + pub dropped: [AtomicUsize; 3], + pub reads: [AtomicUsize; 3], +} + +pub struct TestTransportSource { + pub source: Arc, + pub counts: Arc, +} + +struct TestProcessTransport { + source: Arc, + counts: Arc, + state: AppState, + binding: NodeBinding, + node: NodeEntry, +} + +impl Drop for TestProcessTransport { + fn drop(&mut self) { + self.counts.dropped[usize::try_from(self.node.id).unwrap()].fetch_add(1, Ordering::Relaxed); + } +} + +#[async_trait::async_trait] +impl ProcessObservationForTests for TestProcessTransport { + async fn read(&mut self) -> Result { + self.counts.reads[usize::try_from(self.node.id).unwrap()].fetch_add(1, Ordering::Relaxed); + self.source + .processes(&self.state, &self.binding, &self.node) + .await + } +} + +#[async_trait::async_trait] +impl ServiceObservationSourceForTests for TestTransportSource { + fn process_observer( + &self, + state: &AppState, + binding: &NodeBinding, + node: &NodeEntry, + ) -> Result>, String> { + self.counts.created[usize::try_from(node.id).unwrap()].fetch_add(1, Ordering::Relaxed); + Ok(Some(Box::new(TestProcessTransport { + source: self.source.clone(), + counts: self.counts.clone(), + state: state.clone(), + binding: binding.clone(), + node: node.clone(), + }))) + } + + async fn intents(&self, state: &AppState) -> Result, String> { + self.source.intents(state).await + } + + async fn processes(&self, _: &AppState, _: &NodeBinding, _: &NodeEntry) -> Result { + panic!("the owned process transport must handle process sampling") + } + + async fn health( + &self, + state: &AppState, + config: &ConsoleConfig, + entry: &ServerEntry, + pid: Option, + intents: &[NodeServiceIntent], + ) -> NodeHealth { + self.source.health(state, config, entry, pid, intents).await + } +} diff --git a/app/crowdb-web/tests/common/service_inventory.rs b/app/crowdb-web/tests/common/service_inventory.rs new file mode 100644 index 000000000..db9b2e857 --- /dev/null +++ b/app/crowdb-web/tests/common/service_inventory.rs @@ -0,0 +1,270 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use axum::{ + body::{to_bytes, Body}, + http::Request, +}; +use crowdb_console_shared::{ + cluster::NodeHealth, + config::{NodeEntry, ServerEntry, ServiceType}, + ConsoleConfig, +}; +use crowdb_protocol::mgmt::{ + node::{NodeBinding, NodeServiceAction, NodeServiceIntent}, + SystemBootstrapIdentity, +}; +use crowdb_web::{AppState, ServiceObservationSourceForTests, ServiceObserver}; +use serde_json::{json, Value}; +use std::{ + collections::BTreeMap, + sync::{ + atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::sync::Notify; +use tower::ServiceExt; + +pub struct TestSource { + pub intents: arc_swap::ArcSwap>, + pub fail_authority: AtomicBool, + pub hold_authority: AtomicBool, + pub authority_entered: Notify, + pub authority_release: Notify, + pub authority_finished: Notify, + pub hold_node: AtomicU32, + pub process_entered: Notify, + pub process_release: Notify, + pub process_finished: Notify, + pub hold_health: AtomicBool, + pub health_entered: Notify, + pub health_release: Notify, + pub pid: AtomicU32, + pub wrong_bootstrap: AtomicBool, + pub wrong_operation: AtomicBool, + pub exhausted: AtomicBool, + pub calls: AtomicUsize, +} + +#[async_trait::async_trait] +impl ServiceObservationSourceForTests for TestSource { + async fn intents(&self, _: &AppState) -> Result, String> { + self.calls.fetch_add(1, Ordering::Relaxed); + if self.fail_authority.load(Ordering::Relaxed) { + return Err("fixture authority unavailable".into()); + } + let result = self.intents.load_full().as_ref().clone(); + if self.hold_authority.load(Ordering::Relaxed) { + self.authority_entered.notify_one(); + self.authority_release.notified().await; + self.authority_finished.notify_one(); + } + Ok(result) + } + + async fn processes( + &self, + _: &AppState, + binding: &NodeBinding, + node: &NodeEntry, + ) -> Result { + self.calls.fetch_add(1, Ordering::Relaxed); + let intent = self + .intents + .load_full() + .iter() + .find(|intent| intent.node_id == node.id) + .unwrap() + .clone(); + let pid = self.pid.load(Ordering::Relaxed) + u32::try_from(node.id).unwrap(); + let wrong_operation = self.wrong_operation.load(Ordering::Relaxed); + let mut bootstrap = binding.bootstrap.clone(); + if self.wrong_bootstrap.load(Ordering::Relaxed) { + bootstrap.operation_id = "foreign-bootstrap".into(); + } + if self.hold_node.load(Ordering::Relaxed) == u32::try_from(node.id).unwrap() { + self.process_entered.notify_one(); + self.process_release.notified().await; + self.process_finished.notify_one(); + } + let operation = if wrong_operation { + "old-operation" + } else { + &intent.operation_id + }; + Ok( + json!({"node_id": node.id, "cluster_id": binding.bootstrap.cluster_id, "bootstrap": bootstrap, + "processes": if self.pid.load(Ordering::Relaxed) == 0 { json!({}) } else { json!({intent.service_id.clone(): pid}) }, + "process_operations": {intent.service_id.clone(): operation}, + "service_failures": {intent.service_id: {"operation_id":operation, "exhausted":self.exhausted.load(Ordering::Relaxed)}} }), + ) + } + + async fn health( + &self, + _: &AppState, + _: &ConsoleConfig, + entry: &ServerEntry, + _: Option, + _: &[NodeServiceIntent], + ) -> NodeHealth { + self.calls.fetch_add(1, Ordering::Relaxed); + if self.hold_health.load(Ordering::Relaxed) && entry.id == "chunkdb-sibling" { + self.health_entered.notify_one(); + self.health_release.notified().await; + } + NodeHealth::Up + } +} + +pub struct TestServiceInventory { + pub state: AppState, + pub source: Arc, + pub observer: Option, + pub root: crowdb_test_harness::test_dirs::TestDir, + app: axum::Router, +} + +impl TestServiceInventory { + pub fn new(access: bool) -> Self { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("service-inventory"); + let binding = NodeBinding { + node_id: 1, + bootstrap: SystemBootstrapIdentity { + cluster_id: "inventory-cluster".into(), + operation_id: "bootstrap".into(), + configuration_digest: "a".repeat(64), + }, + management_seeds: vec!["http://127.0.0.1:1".into()], + }; + std::fs::write( + root.path().join("node-binding.json"), + serde_json::to_vec(&binding).unwrap(), + ) + .unwrap(); + let mut config = ConsoleConfig::default(); + let mut intents = Vec::new(); + for node_id in [1, 2] { + config.nodes.push(NodeEntry { + id: node_id, + rack_id: 1, + host: format!("127.0.0.{node_id}"), + ssh_port: 22, + ssh_user: "fixture".into(), + ssh_credential_ref: None, + ssh_key: None, + ssh_password: None, + }); + let id = format!("{}-{node_id}", if access { "access" } else { "chunkdb" }); + let mut entry = ServerEntry::new(&id, format!("http://127.0.0.{node_id}:7300")); + entry.node_id = Some(node_id); + entry.service_type = if access { + ServiceType::AccessServer + } else { + ServiceType::Chunkdb + }; + entry.auto_start = true; + entry.rest_port = Some(7300); + if access { + entry.access_origins.insert("iceberg".into(), entry.url.clone()); + } else { + entry.rpc_url = Some(format!("127.0.0.{node_id}:7400")); + entry.rpc_port = Some(7400); + } + config.servers.push(entry); + intents.push(NodeServiceIntent { cluster_id: binding.bootstrap.cluster_id.clone(), + operation_id: format!("operation-{node_id}"), node_id, service_id: id, kind: if access { "access" } else { "chunkdb" }.into(), action: NodeServiceAction::Start, + configuration: if access { format!("[iceberg]\nlisten = '127.0.0.{node_id}:7300'\n[health]\nlisten = '127.0.0.{node_id}:7500'\n") } + else { format!("[server]\nhttp_listen_addr = '127.0.0.{node_id}:7300'\nrpc_listen_addr = '127.0.0.{node_id}:7400'\n") }, + environment: BTreeMap::default() }); + } + let state = AppState::with_runtime_root(config, root.path().join("console")) + .with_node_monitor("http://127.0.0.1:1") + .unwrap(); + let app = crowdb_web::router(state.clone()); + let source = Arc::new(TestSource { + intents: arc_swap::ArcSwap::from_pointee(intents), + fail_authority: AtomicBool::new(false), + hold_authority: AtomicBool::new(false), + authority_entered: Notify::new(), + authority_release: Notify::new(), + authority_finished: Notify::new(), + hold_node: AtomicU32::new(0), + process_entered: Notify::new(), + process_release: Notify::new(), + process_finished: Notify::new(), + hold_health: AtomicBool::new(false), + health_entered: Notify::new(), + health_release: Notify::new(), + pid: AtomicU32::new(100), + wrong_bootstrap: AtomicBool::new(false), + wrong_operation: AtomicBool::new(false), + exhausted: AtomicBool::new(false), + calls: AtomicUsize::new(0), + }); + Self { + state, + source, + observer: None, + root, + app, + } + } + + pub fn start(&mut self) { + self.observer = + Some(crowdb_web::start_service_observer_for_tests(&self.state, self.source.clone()).unwrap()); + } + + pub async fn rows(&self) -> Value { + let response = tokio::time::timeout( + Duration::from_millis(200), + self.app.clone().oneshot( + Request::builder() + .uri("/api/servers") + .body(Body::empty()) + .unwrap(), + ), + ) + .await + .expect("inventory GET waited for authority, SSH, or endpoint work") + .unwrap(); + assert!(response.status().is_success()); + serde_json::from_slice(&to_bytes(response.into_body(), 65_536).await.unwrap()).unwrap() + } + + pub async fn wait_pid(&self, node: u64, pid: Option) -> Value { + let id = self + .source + .intents + .load_full() + .iter() + .find(|intent| intent.node_id == node) + .map(|intent| intent.service_id.clone()); + tokio::time::timeout(Duration::from_secs(2), async { + loop { + let rows = self.rows().await; + let row = rows + .as_array() + .unwrap() + .iter() + .find(|row| row["node_id"] == node && id.as_ref().map_or(true, |id| row["id"] == *id)) + .unwrap(); + if row["pid"].as_u64() == pid.map(u64::from) { + return row.clone(); + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("sampler did not publish the expected monitor PID") + } + + pub async fn signalled(signal: &Notify) { + tokio::time::timeout(Duration::from_secs(2), signal.notified()) + .await + .expect("observation source did not reach its deterministic barrier"); + } +} diff --git a/app/crowdb-web/tests/diskdb_deployment_configuration_test.rs b/app/crowdb-web/tests/diskdb_deployment_configuration_test.rs index 8fba4db38..335a7c1a2 100644 --- a/app/crowdb-web/tests/diskdb_deployment_configuration_test.rs +++ b/app/crowdb-web/tests/diskdb_deployment_configuration_test.rs @@ -9,6 +9,44 @@ use crowdb_web::{router, AppState}; use serde_json::{json, Value}; use tower::ServiceExt; +#[test] +fn explicit_diskdb_ranges_keep_adjacent_host_instances_disjoint_and_preserve_legacy_inputs() { + use crowdb_console_shared::config::NodeEntry; + use crowdb_web::diskdb::{configuration_for_tests, DeployDiskdbBody}; + let node: NodeEntry = serde_json::from_value(json!({ + "id":1,"rack_id":1,"host":"127.0.0.1","ssh_port":2222,"ssh_user":"crowdb" + })) + .unwrap(); + let mut used = std::collections::BTreeSet::new(); + for offset in 0..3 { + let body: DeployDiskdbBody = serde_json::from_value(json!({ + "listen_port":11000+offset,"http_port":11100+offset, + "rpc_port":11200+offset,"rpc_listen_port":11200+offset + })) + .unwrap(); + let config = configuration_for_tests(&node, &["http://127.0.0.1:10000".into()], &body) + .unwrap_or_else(|(_, body)| panic!("{}", body.error)); + let config: toml::Value = toml::from_str(&config).unwrap(); + for (field, base) in [ + ("listen_addr", 11000), + ("http_listen_addr", 11100), + ("rpc_listen_addr", 11200), + ] { + let address = config["server"][field].as_str().unwrap(); + assert_eq!(address, format!("127.0.0.1:{}", base + offset)); + assert!(used.insert(address.to_owned())); + } + } + let legacy: DeployDiskdbBody = serde_json::from_value(json!({"rpc_port":7100})).unwrap(); + let config = + configuration_for_tests(&node, &[], &legacy).unwrap_or_else(|(_, body)| panic!("{}", body.error)); + let config: toml::Value = toml::from_str(&config).unwrap(); + assert_eq!( + config["server"]["rpc_listen_addr"].as_str(), + Some("127.0.0.1:7102") + ); +} + async fn submit(body: Value) -> (StatusCode, String) { let state = AppState::default(); let response = router(state) diff --git a/app/crowdb-web/tests/internal/service_start_pending_test.rs b/app/crowdb-web/tests/internal/service_start_pending_test.rs deleted file mode 100644 index 390aeedea..000000000 --- a/app/crowdb-web/tests/internal/service_start_pending_test.rs +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -use super::{NodeServiceAction, NodeServiceIntent, ProcessObservations, ServerEntry}; -use serde_json::json; - -#[test] -fn accepted_start_waits_until_owned_pid_or_current_operation_exhaustion() { - let intent = NodeServiceIntent { - node_id: 2, - service_id: "chunkdb-7".into(), - kind: "chunkdb".into(), - cluster_id: "cluster".into(), - operation_id: "current".into(), - action: NodeServiceAction::Start, - configuration: String::new(), - environment: std::collections::BTreeMap::new(), - }; - let mut entry = ServerEntry::new("chunkdb-7", "http://127.0.0.1:7300"); - entry.node_id = Some(2); - entry.auto_start = true; - for (operation, exhausted, pending) in [ - ("old", true, true), - ("current", false, true), - ("current", true, false), - ] { - let mut observation = ProcessObservations::default(); - observation.record( - 2, - &json!({"processes":{}, "service_failures": { - "chunkdb-7": {"operation_id": operation, "exhausted": exhausted} - }}), - std::slice::from_ref(&intent), - ); - assert_eq!(observation.start_pending(&entry, true, None), pending); - assert!(!observation.start_pending(&entry, true, Some(42))); - assert!(!observation.start_pending(&entry, false, None)); - } - entry.auto_start = false; - assert!(!ProcessObservations::default().start_pending(&entry, true, None)); -} diff --git a/app/crowdb-web/tests/node_confirmed_projection_test.rs b/app/crowdb-web/tests/node_confirmed_projection_test.rs new file mode 100644 index 000000000..fb62cb88f --- /dev/null +++ b/app/crowdb-web/tests/node_confirmed_projection_test.rs @@ -0,0 +1,60 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_console_shared::{ + config::{ServerEntry, ServiceType}, + deployment::registry::NodeRecord, +}; +use crowdb_web::project_confirmed_servers_for_tests; + +#[test] +fn auxiliary_projection_before_confirmation_keeps_bootstrap_kv_and_accepted_ports() { + for host in ["192.0.2.42", "2001:db8::42"] { + let mut auxiliary = ServerEntry::new("access-42", "http://original:16000"); + auxiliary.node_id = Some(42); + auxiliary.service_type = ServiceType::AccessServer; + auxiliary.rpc_url = Some("original:16001".into()); + let mut kv = ServerEntry::new("kv-42", "http://original:10042"); + kv.node_id = Some(42); + kv.rpc_url = Some("original:10142".into()); + let nodes = vec![NodeRecord { + discovery_id: "discovery-42".into(), + node_id: 42, + physical_host_id: "host-42".into(), + rack_id: 1, + host: host.into(), + web_port: 9090, + ssh_port: 22, + ssh_user: "crowdb".into(), + operation_id: "admit-42".into(), + host_key: None, + confirmed: true, + cancelled: false, + }]; + let mut servers = vec![auxiliary.clone()]; + project_confirmed_servers_for_tests(&mut servers, vec![kv.clone()], &nodes); + assert_eq!( + servers.len(), + 2, + "auxiliary publication must not hide bootstrap KV" + ); + assert_eq!( + servers[0], auxiliary, + "KV recovery must not rewrite an auxiliary endpoint" + ); + let host = if host.contains(':') { + format!("[{host}]") + } else { + host.into() + }; + kv.url = format!("http://{host}:10042"); + kv.rpc_url = Some(format!("{host}:10142")); + assert_eq!(servers[1], kv); + let retained = servers.clone(); + project_confirmed_servers_for_tests(&mut servers, vec![kv], &nodes); + assert_eq!( + servers, retained, + "repeated confirmation must preserve service identities" + ); + } +} diff --git a/app/crowdb-web/tests/node_listener_recovery_test.rs b/app/crowdb-web/tests/node_listener_recovery_test.rs index bc0657501..3eb38f6bd 100644 --- a/app/crowdb-web/tests/node_listener_recovery_test.rs +++ b/app/crowdb-web/tests/node_listener_recovery_test.rs @@ -27,6 +27,7 @@ async fn background_protocol_reads_follow_confirmed_addresses_without_ui_refresh let registry = NodeRegistry { cluster_id: fixture.binding.bootstrap.cluster_id.clone(), nodes: vec![NodeRecord { + web_port: 9090, discovery_id: discovery.clone(), node_id: 2, physical_host_id: "retained-host".into(), diff --git a/app/crowdb-web/tests/node_preflight_test.rs b/app/crowdb-web/tests/node_preflight_test.rs index db4b955e6..5d048132e 100644 --- a/app/crowdb-web/tests/node_preflight_test.rs +++ b/app/crowdb-web/tests/node_preflight_test.rs @@ -19,6 +19,7 @@ async fn a_preflight_review_is_not_admission_and_discard_is_operation_fenced_and let discovery = uuid::Uuid::new_v4().to_string(); let operation = uuid::Uuid::new_v4().to_string(); let record = NodeRecord { + web_port: 9090, discovery_id: discovery.clone(), operation_id: operation.clone(), node_id: 1, diff --git a/app/crowdb-web/tests/remote_kv_projection_test.rs b/app/crowdb-web/tests/remote_kv_projection_test.rs index 2953c2a5d..864cec16e 100644 --- a/app/crowdb-web/tests/remote_kv_projection_test.rs +++ b/app/crowdb-web/tests/remote_kv_projection_test.rs @@ -99,6 +99,14 @@ async fn kv_lifecycle_intents_preserve_admitted_endpoints_while_auxiliary_servic .with_node_monitor("http://127.0.0.1:12345") .unwrap(); *state.kv_client.write().await = Some(kv); + let _observer = crowdb_web::start_service_observer(&state).unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(3), async { + while state.config.read().unwrap().servers.len() != 2 { + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + }) + .await + .expect("background authority projection did not publish auxiliary registration"); let response = router(state) .oneshot( Request::builder() diff --git a/app/crowdb-web/tests/s3_inspection_proxy_test.rs b/app/crowdb-web/tests/s3_inspection_proxy_test.rs index 9df5cc72b..ea856bbdf 100644 --- a/app/crowdb-web/tests/s3_inspection_proxy_test.rs +++ b/app/crowdb-web/tests/s3_inspection_proxy_test.rs @@ -30,7 +30,7 @@ async fn upstream( r#"{"error":"Object generation changed"}"#.into(), ), "missing" => (StatusCode::NOT_FOUND, r#"{"error":"Object missing"}"#.into()), - "folder/中文.bin" => { + "folder/\u{4e2d}\u{6587}.bin" => { assert_eq!(query["limit"], "20"); assert_eq!(query["cursor"], "opaque+/="); (StatusCode::OK, r#"{"offset":"9007199254740993"}"#.into()) diff --git a/app/crowdb-web/tests/service_inventory_observation_test.rs b/app/crowdb-web/tests/service_inventory_observation_test.rs new file mode 100644 index 000000000..1c177d022 --- /dev/null +++ b/app/crowdb-web/tests/service_inventory_observation_test.rs @@ -0,0 +1,241 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/service_inventory.rs"] +mod fixture; +use crowdb_protocol::mgmt::node::NodeServiceAction; +use fixture::TestServiceInventory; +use serde_json::json; +use std::{ + sync::{atomic::Ordering, Arc}, + time::Duration, +}; + +#[tokio::test] +async fn inventory_readers_do_not_wait_for_an_unrelated_monitor_or_refresh_evidence_age() { + let mut fixture = TestServiceInventory::new(false); + fixture.source.hold_node.store(2, Ordering::Relaxed); + fixture.start(); + TestServiceInventory::signalled(&fixture.source.process_entered).await; + let row = fixture.wait_pid(1, Some(101)).await; + assert_eq!(row["pid_node_id"], 1); + assert_eq!(row["process_observed"], true); + assert_eq!(fixture.rows().await.as_array().unwrap().len(), 2); + assert!(crowdb_web::start_service_observer(&fixture.state).is_err()); + fixture.observer.take(); + tokio::task::yield_now().await; + let calls = fixture.source.calls.load(Ordering::Relaxed); + for rows in futures::future::join_all((0..32).map(|_| fixture.rows())).await { + assert_eq!(rows[0]["pid"], 101); + } + assert_eq!(fixture.source.calls.load(Ordering::Relaxed), calls); + tokio::time::sleep(Duration::from_millis(3050)).await; + let rows = fixture.rows().await; + assert!(rows[0]["pid"].is_null()); + assert!(rows[0]["pid_node_id"].is_null()); + assert_eq!(rows[0]["health"], "unknown"); + assert_eq!(rows[0]["process_observed"], false); + assert_eq!(fixture.source.calls.load(Ordering::Relaxed), calls); +} + +#[tokio::test] +async fn a_late_pre_mutation_process_sample_cannot_replace_a_confirmed_pid_or_hide_siblings() { + let mut fixture = TestServiceInventory::new(false); + fixture.start(); + fixture.wait_pid(1, Some(101)).await; + fixture.wait_pid(2, Some(102)).await; + fixture.source.hold_node.store(1, Ordering::Relaxed); + TestServiceInventory::signalled(&fixture.source.process_entered).await; + let intent = fixture.source.intents.load_full()[0].clone(); + let mutation = + crowdb_web::invalidate_service_observation_for_tests(&fixture.state, 1, &intent.service_id); + let invalidated = fixture.rows().await; + assert!(invalidated + .as_array() + .unwrap() + .iter() + .find(|row| row["node_id"] == 1) + .unwrap()["pid"] + .is_null()); + assert_eq!(fixture.wait_pid(2, Some(102)).await["pid"], 102); + // The same durable operation may resume with a replacement process. Its + // epoch still fences a delayed observation with the identical operation ID. + fixture.source.pid.store(500, Ordering::Relaxed); + mutation.complete( + intent.clone(), + &json!({"operation_id":intent.operation_id,"pid":501}), + ); + assert_eq!(fixture.wait_pid(1, Some(501)).await["pid"], 501); + fixture.source.hold_node.store(0, Ordering::Relaxed); + fixture.source.process_release.notify_one(); + TestServiceInventory::signalled(&fixture.source.process_finished).await; + for _ in 0..32 { + assert_eq!(fixture.wait_pid(1, Some(501)).await["pid"], 501); + tokio::task::yield_now().await; + } +} + +#[tokio::test] +async fn failed_or_unknown_mutation_does_not_publish_an_assumed_pid() { + let mut fixture = TestServiceInventory::new(false); + fixture.start(); + fixture.wait_pid(1, Some(101)).await; + fixture.source.hold_node.store(1, Ordering::Relaxed); + TestServiceInventory::signalled(&fixture.source.process_entered).await; + let mutation = crowdb_web::invalidate_service_observation_for_tests(&fixture.state, 1, "chunkdb-1"); + drop(mutation); + assert!(fixture + .rows() + .await + .as_array() + .unwrap() + .iter() + .find(|row| row["node_id"] == 1) + .unwrap()["pid"] + .is_null()); + fixture.source.pid.store(800, Ordering::Relaxed); + fixture.source.hold_node.store(0, Ordering::Relaxed); + fixture.source.process_release.notify_one(); + TestServiceInventory::signalled(&fixture.source.process_finished).await; + fixture.wait_pid(1, Some(801)).await; +} + +#[tokio::test] +async fn an_unknown_outcome_also_fences_samples_started_during_the_mutation() { + let mut fixture = TestServiceInventory::new(false); + fixture.start(); + fixture.wait_pid(1, Some(101)).await; + let mutation = crowdb_web::invalidate_service_observation_for_tests(&fixture.state, 1, "chunkdb-1"); + fixture.source.hold_node.store(1, Ordering::Relaxed); + TestServiceInventory::signalled(&fixture.source.process_entered).await; + fixture.source.pid.store(800, Ordering::Relaxed); + drop(mutation); + fixture.source.hold_node.store(0, Ordering::Relaxed); + fixture.source.process_release.notify_one(); + TestServiceInventory::signalled(&fixture.source.process_finished).await; + let rows = fixture.rows().await; + assert!(rows + .as_array() + .unwrap() + .iter() + .find(|row| row["node_id"] == 1) + .unwrap()["pid"] + .is_null()); + fixture.wait_pid(1, Some(801)).await; +} + +#[tokio::test] +async fn a_slow_sibling_health_probe_does_not_delay_target_process_or_health_resampling() { + let mut fixture = TestServiceInventory::new(false); + let mut sibling = fixture.state.config.read().unwrap().servers[0].clone(); + sibling.id = "chunkdb-sibling".into(); + fixture.state.config.write().unwrap().servers.push(sibling); + fixture.source.hold_health.store(true, Ordering::Relaxed); + fixture.start(); + TestServiceInventory::signalled(&fixture.source.health_entered).await; + fixture.wait_pid(1, Some(101)).await; + let intent = fixture.source.intents.load_full()[0].clone(); + let mutation = + crowdb_web::invalidate_service_observation_for_tests(&fixture.state, 1, &intent.service_id); + fixture.source.pid.store(500, Ordering::Relaxed); + mutation.complete( + intent.clone(), + &json!({"operation_id":intent.operation_id,"pid":501}), + ); + drop(mutation); + tokio::time::timeout(Duration::from_secs(2), async { + loop { + let row = fixture.wait_pid(1, Some(501)).await; + if row["health"] == "up" { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("target readiness waited behind a sibling health probe"); +} + +#[tokio::test] +async fn fresh_bound_processes_remain_observable_when_authority_expires_but_access_health_is_unknown() { + let mut fixture = TestServiceInventory::new(true); + fixture.start(); + fixture.wait_pid(1, Some(101)).await; + fixture.source.fail_authority.store(true, Ordering::Relaxed); + tokio::time::sleep(Duration::from_millis(3150)).await; + let row = fixture.wait_pid(1, Some(101)).await; + assert_eq!(row["pid_node_id"], 1); + assert_eq!(row["process_observed"], true); + assert_eq!(row["health"], "unknown"); + assert_eq!(row["start_pending"], false); +} + +#[tokio::test] +async fn changed_endpoint_wrong_bootstrap_and_wrong_executed_operation_never_reuse_a_pid() { + let mut fixture = TestServiceInventory::new(false); + fixture.start(); + fixture.wait_pid(1, Some(101)).await; + fixture.source.wrong_bootstrap.store(true, Ordering::Relaxed); + fixture.wait_pid(1, None).await; + fixture.source.wrong_bootstrap.store(false, Ordering::Relaxed); + fixture.source.wrong_operation.store(true, Ordering::Relaxed); + tokio::time::sleep(Duration::from_millis(1100)).await; + assert!(fixture.wait_pid(1, None).await["pid_node_id"].is_null()); + fixture.source.wrong_operation.store(false, Ordering::Relaxed); + fixture.wait_pid(1, Some(101)).await; + fixture.state.config.write().unwrap().nodes[0].host = "127.0.0.99".into(); + let rows = fixture.rows().await; + let row = rows + .as_array() + .unwrap() + .iter() + .find(|row| row["node_id"] == 1) + .unwrap(); + assert!(row["pid"].is_null()); + assert_eq!(row["process_observed"], false); + assert_eq!(row["health"], "unknown"); + std::fs::remove_file(fixture.root.path().join("node-binding.json")).unwrap(); + for row in fixture.rows().await.as_array().unwrap() { + assert!(row["pid"].is_null()); + } +} + +#[tokio::test] +async fn pending_start_uses_only_the_current_operation_exhaustion() { + let mut fixture = TestServiceInventory::new(false); + fixture.source.pid.store(0, Ordering::Relaxed); + fixture.source.exhausted.store(true, Ordering::Relaxed); + fixture.source.wrong_operation.store(true, Ordering::Relaxed); + fixture.start(); + tokio::time::sleep(Duration::from_millis(150)).await; + assert_eq!(fixture.wait_pid(1, None).await["start_pending"], true); + fixture.source.wrong_operation.store(false, Ordering::Relaxed); + tokio::time::sleep(Duration::from_millis(1100)).await; + assert_eq!(fixture.wait_pid(1, None).await["start_pending"], false); +} + +#[tokio::test] +async fn a_late_authority_sample_cannot_resurrect_a_deleted_registration() { + let mut fixture = TestServiceInventory::new(false); + fixture.start(); + fixture.wait_pid(1, Some(101)).await; + fixture.source.hold_authority.store(true, Ordering::Relaxed); + TestServiceInventory::signalled(&fixture.source.authority_entered).await; + let mutation = crowdb_web::invalidate_service_observation_for_tests(&fixture.state, 1, "chunkdb-1"); + let mut intents = fixture.source.intents.load_full().as_ref().clone(); + intents[0].action = NodeServiceAction::Delete; + intents[0].operation_id = "delete-operation".into(); + fixture.source.intents.store(Arc::new(intents.clone())); + mutation.complete( + intents[0].clone(), + &json!({"operation_id":"delete-operation","pid":null}), + ); + assert_eq!(fixture.rows().await.as_array().unwrap().len(), 1); + fixture.source.hold_authority.store(false, Ordering::Relaxed); + fixture.source.authority_release.notify_one(); + TestServiceInventory::signalled(&fixture.source.authority_finished).await; + for _ in 0..32 { + assert_eq!(fixture.rows().await.as_array().unwrap().len(), 1); + tokio::task::yield_now().await; + } +} diff --git a/app/crowdb-web/tests/service_process_observed_test.rs b/app/crowdb-web/tests/service_process_observed_test.rs new file mode 100644 index 000000000..d818d4de3 --- /dev/null +++ b/app/crowdb-web/tests/service_process_observed_test.rs @@ -0,0 +1,230 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/process_observed.rs"] +mod evidence; +#[path = "common/service_inventory.rs"] +mod fixture; + +use crowdb_console_shared::{config::ServerEntry, ConsoleConfig}; +use crowdb_web::AppState; +use evidence::{ + start, wait, ABSENT, ARRAY_MAP, EXHAUSTED, FOREIGN_BOOTSTRAP, FOREIGN_CLUSTER, FOREIGN_EXHAUSTION, + FOREIGN_NODE, FOREIGN_OPERATION, MISSING_MAP, NULL_MAP, READ_ERROR, RUNNING, STRING_PID, ZERO_PID, +}; +use fixture::TestServiceInventory; +use serde_json::{json, Value}; +use std::{sync::atomic::Ordering, time::Duration}; + +#[tokio::test] +async fn unavailable_process_evidence_is_unknown_even_when_endpoint_health_is_up_then_same_pid_returns() { + let mut fixture = TestServiceInventory::new(false); + let initial = fixture.rows().await; + assert_eq!(initial[0]["process_observed"], false); + let mode = start(&mut fixture, false); + assert_eq!(wait(&fixture, "running").await["pid"], 101); + mode.store(READ_ERROR, Ordering::Relaxed); + let row = wait(&fixture, "unknown").await; + assert!(row["pid"].is_null()); + assert_eq!(row["health"], "up"); + mode.store(RUNNING, Ordering::Relaxed); + assert_eq!(wait(&fixture, "running").await["pid"], 101); +} + +#[tokio::test] +async fn fresh_empty_process_map_confirms_stopped_and_only_current_exhaustion_ends_pending_start() { + let mut fixture = TestServiceInventory::new(false); + let mode = start(&mut fixture, false); + wait(&fixture, "running").await; + mode.store(ABSENT, Ordering::Relaxed); + let row = wait(&fixture, "stopped").await; + assert!(row["pid"].is_null()); + assert_eq!(row["start_pending"], true); + mode.store(EXHAUSTED, Ordering::Relaxed); + tokio::time::timeout(Duration::from_secs(2), async { + loop { + let row = wait(&fixture, "stopped").await; + if row["start_pending"] == false { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .unwrap(); + mode.store(FOREIGN_EXHAUSTION, Ordering::Relaxed); + tokio::time::timeout(Duration::from_secs(2), async { + loop { + let row = wait(&fixture, "stopped").await; + if row["start_pending"] == true { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn foreign_live_operation_or_bootstrap_is_unknown_never_observed_stopped_or_adopted() { + let mut fixture = TestServiceInventory::new(false); + let mode = start(&mut fixture, false); + wait(&fixture, "running").await; + for wrong in [FOREIGN_OPERATION, FOREIGN_BOOTSTRAP] { + mode.store(wrong, Ordering::Relaxed); + let row = wait(&fixture, "unknown").await; + assert!(row["pid"].is_null()); + mode.store(RUNNING, Ordering::Relaxed); + assert_eq!(wait(&fixture, "running").await["pid"], 101); + } +} + +#[tokio::test] +async fn expired_process_evidence_is_unknown_and_never_retains_a_pid() { + let mut fixture = TestServiceInventory::new(false); + start(&mut fixture, false); + wait(&fixture, "running").await; + fixture.observer.take(); + tokio::task::yield_now().await; + tokio::time::sleep(Duration::from_millis(3050)).await; + let row = wait(&fixture, "unknown").await; + assert!(row["pid"].is_null()); + assert_eq!(row["start_pending"], false); +} + +#[tokio::test] +async fn bound_bootstrap_pkv_without_intent_distinguishes_no_read_from_positive_process_absence() { + let mut fixture = TestServiceInventory::new(false); + let mode = start(&mut fixture, true); + let row = wait(&fixture, "running").await; + assert_eq!(row["pid"], 101); + assert_eq!(row["start_pending"], false); + mode.store(ABSENT, Ordering::Relaxed); + let row = wait(&fixture, "stopped").await; + assert!(row["pid"].is_null()); + assert_eq!(row["start_pending"], false); + mode.store(READ_ERROR, Ordering::Relaxed); + assert!(wait(&fixture, "unknown").await["pid"].is_null()); + mode.store(RUNNING, Ordering::Relaxed); + assert_eq!(wait(&fixture, "running").await["pid"], 101); +} + +#[tokio::test] +async fn missing_malformed_process_maps_and_invalid_pid_values_never_confirm_absence() { + let mut fixture = TestServiceInventory::new(false); + let mode = start(&mut fixture, false); + wait(&fixture, "running").await; + for malformed in [MISSING_MAP, NULL_MAP, ARRAY_MAP, ZERO_PID, STRING_PID] { + mode.store(malformed, Ordering::Relaxed); + assert!(wait(&fixture, "unknown").await["pid"].is_null()); + mode.store(RUNNING, Ordering::Relaxed); + assert_eq!(wait(&fixture, "running").await["pid"], 101); + } +} + +#[tokio::test] +async fn wrong_monitor_node_or_cluster_is_unknown_not_observed_stopped() { + let mut fixture = TestServiceInventory::new(false); + let mode = start(&mut fixture, false); + wait(&fixture, "running").await; + for foreign in [FOREIGN_NODE, FOREIGN_CLUSTER] { + mode.store(foreign, Ordering::Relaxed); + assert!(wait(&fixture, "unknown").await["pid"].is_null()); + mode.store(RUNNING, Ordering::Relaxed); + assert_eq!(wait(&fixture, "running").await["pid"], 101); + } +} + +#[tokio::test] +async fn accepted_no_pid_mutation_reply_does_not_manufacture_process_absence() { + let mut fixture = TestServiceInventory::new(false); + fixture.start(); + fixture.wait_pid(1, Some(101)).await; + fixture.source.hold_node.store(1, Ordering::Relaxed); + TestServiceInventory::signalled(&fixture.source.process_entered).await; + let intent = fixture.source.intents.load_full()[0].clone(); + let mutation = + crowdb_web::invalidate_service_observation_for_tests(&fixture.state, 1, &intent.service_id); + mutation.complete( + intent.clone(), + &json!({"operation_id": intent.operation_id, "pid": null}), + ); + let row = wait(&fixture, "unknown").await; + assert!(row["pid"].is_null()); + assert_eq!(row["start_pending"], true); + fixture.source.hold_node.store(0, Ordering::Relaxed); + fixture.source.process_release.notify_one(); + TestServiceInventory::signalled(&fixture.source.process_finished).await; + fixture.wait_pid(1, Some(101)).await; + assert_eq!(wait(&fixture, "running").await["pid"], 101); +} + +#[tokio::test] +async fn accepted_actual_pid_mutation_reply_is_observed_and_late_sample_remains_fenced() { + let mut fixture = TestServiceInventory::new(false); + fixture.start(); + fixture.wait_pid(1, Some(101)).await; + fixture.source.hold_node.store(1, Ordering::Relaxed); + TestServiceInventory::signalled(&fixture.source.process_entered).await; + let intent = fixture.source.intents.load_full()[0].clone(); + let mutation = + crowdb_web::invalidate_service_observation_for_tests(&fixture.state, 1, &intent.service_id); + fixture.source.pid.store(500, Ordering::Relaxed); + mutation.complete( + intent.clone(), + &json!({"operation_id": intent.operation_id, "pid": 501}), + ); + let row = wait(&fixture, "running").await; + assert_eq!(row["pid"], 501); + fixture.source.hold_node.store(0, Ordering::Relaxed); + fixture.source.process_release.notify_one(); + TestServiceInventory::signalled(&fixture.source.process_finished).await; + assert_eq!(wait(&fixture, "running").await["pid"], 501); +} + +#[tokio::test] +async fn changed_node_configuration_or_missing_binding_never_reuses_observed_process_evidence() { + let mut fixture = TestServiceInventory::new(false); + start(&mut fixture, false); + wait(&fixture, "running").await; + // Freeze the accepted old fact; a legitimate new-node read is outside this check. + fixture.observer.take(); + tokio::task::yield_now().await; + fixture.state.config.write().unwrap().nodes[0].host = "127.0.0.99".into(); + let row = wait(&fixture, "unknown").await; + assert!(row["pid"].is_null()); + std::fs::remove_file(fixture.root.path().join("node-binding.json")).unwrap(); + for row in fixture.rows().await.as_array().unwrap() { + assert_eq!(row["process_observed"], false); + assert!(row["pid"].is_null()); + } +} + +#[tokio::test] +async fn local_inventory_omits_monitor_observation_validity_for_legacy_consumers() { + use axum::{ + body::{to_bytes, Body}, + http::Request, + }; + use tower::ServiceExt; + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("local-process-observed"); + let mut config = ConsoleConfig::default(); + config + .servers + .push(ServerEntry::new("external", "http://127.0.0.1:1")); + let state = AppState::with_runtime_root(config, root.path().join("console")); + let response = crowdb_web::router(state) + .oneshot( + Request::builder() + .uri("/api/servers") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert!(response.status().is_success()); + let rows: Value = serde_json::from_slice(&to_bytes(response.into_body(), 65_536).await.unwrap()).unwrap(); + assert_eq!(rows.as_array().unwrap().len(), 1); + assert!(!rows[0].as_object().unwrap().contains_key("process_observed")); +} diff --git a/app/crowdb-web/tests/service_process_transport_test.rs b/app/crowdb-web/tests/service_process_transport_test.rs new file mode 100644 index 000000000..092e25977 --- /dev/null +++ b/app/crowdb-web/tests/service_process_transport_test.rs @@ -0,0 +1,67 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/service_inventory.rs"] +#[allow(dead_code)] // Shared fixture also supplies entry points used by the inventory suite. +mod fixture; +#[path = "common/process_transport.rs"] +mod transport; + +use std::{ + sync::{atomic::Ordering, Arc}, + time::Duration, +}; +use transport::{TestCounts, TestTransportSource}; + +async fn until(condition: impl Fn() -> bool) { + tokio::time::timeout(Duration::from_secs(2), async { + while !condition() { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("process transport ownership did not converge"); +} + +#[tokio::test] +async fn process_transport_belongs_to_one_node_configuration_and_bootstrap() { + let mut fixture = fixture::TestServiceInventory::new(false); + let counts = Arc::new(TestCounts::default()); + let source = Arc::new(TestTransportSource { + source: fixture.source.clone(), + counts: counts.clone(), + }); + fixture.observer = Some(crowdb_web::start_service_observer_for_tests(&fixture.state, source).unwrap()); + fixture.wait_pid(1, Some(101)).await; + fixture.wait_pid(2, Some(102)).await; + until(|| counts.reads[2].load(Ordering::Relaxed) >= 2).await; + assert_eq!(counts.created[1].load(Ordering::Relaxed), 1); + assert_eq!(counts.created[2].load(Ordering::Relaxed), 1); + fixture.state.config.write().unwrap().nodes[1].host = "127.0.0.99".into(); + until(|| { + counts.created[2].load(Ordering::Relaxed) == 2 && counts.dropped[2].load(Ordering::Relaxed) == 1 + }) + .await; + assert_eq!(counts.created[1].load(Ordering::Relaxed), 1); + let path = fixture.root.path().join("node-binding.json"); + let mut binding: crowdb_protocol::mgmt::node::NodeBinding = + serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + std::fs::remove_file(&path).unwrap(); + until(|| { + counts.dropped[1].load(Ordering::Relaxed) == 1 && counts.dropped[2].load(Ordering::Relaxed) == 2 + }) + .await; + binding.bootstrap.operation_id = "replacement-bootstrap".into(); + std::fs::write(&path, serde_json::to_vec(&binding).unwrap()).unwrap(); + fixture.wait_pid(1, Some(101)).await; + fixture.wait_pid(2, Some(102)).await; + assert_eq!(counts.created[1].load(Ordering::Relaxed), 2); + assert_eq!(counts.created[2].load(Ordering::Relaxed), 3); + fixture.observer.take(); + until(|| { + (1..=2).all(|node| { + counts.created[node].load(Ordering::Relaxed) == counts.dropped[node].load(Ordering::Relaxed) + }) + }) + .await; +} diff --git a/app/crowdb-web/ui/src/api.ts b/app/crowdb-web/ui/src/api.ts index d9bb422fa..e6bf57ee5 100644 --- a/app/crowdb-web/ui/src/api.ts +++ b/app/crowdb-web/ui/src/api.ts @@ -1197,6 +1197,8 @@ export async function removeDisk(nodeId: number, dgId: number, diskId: string, o /** `GET /api/servers` — list all deployed server entries. */ export interface ServerSummary { + /** Fresh authenticated process evidence; omitted by legacy local inventories. */ + process_observed?: boolean; start_pending?: boolean; id?: string; node_id?: number; diff --git a/app/crowdb-web/ui/src/dataset/editor/Table.test.tsx b/app/crowdb-web/ui/src/dataset/editor/Table.test.tsx index c5f8f41f3..17b164edd 100644 --- a/app/crowdb-web/ui/src/dataset/editor/Table.test.tsx +++ b/app/crowdb-web/ui/src/dataset/editor/Table.test.tsx @@ -43,7 +43,7 @@ it('edits a Sample as one row, persists only metadata and does not publish on Sa it('patches a cell without losing other Fields and rejects unbounded authoring input', () => { const op = setCell(draft(), btoa('note'), 'label', textValue('keep')); expect(setCell(op, btoa('note'), 'content', { kind: 'upload', id: '3'.repeat(32), length: '12', md5: '4'.repeat(32) }).mapping!.samples[0]).toMatchObject({ fields: [{ name: 'label' }, { name: 'content' }] }); - expect(() => textValue('字'.repeat(22000))).toThrow('64 KiB'); + expect(() => textValue('\u5b57'.repeat(22000))).toThrow('64 KiB'); expect(() => checkLimits({ ...op, mapping: { ...op.mapping!, samples: Array(1001).fill(op.mapping!.samples[0]) } })).toThrow('1000'); }); it('projects new columns as Missing without asking the parent for unknown Fields', async () => { diff --git a/app/crowdb-web/ui/src/kv/displayBytes.test.ts b/app/crowdb-web/ui/src/kv/displayBytes.test.ts index 53c72e1d9..78c8dc96f 100644 --- a/app/crowdb-web/ui/src/kv/displayBytes.test.ts +++ b/app/crowdb-web/ui/src/kv/displayBytes.test.ts @@ -7,7 +7,7 @@ describe('mixed KV byte presentation', () => { it('keeps printable fields around control and invalid UTF-8 bytes', () => { expect(byteRuns(undefined, '616200ffe4b8ad6364')).toEqual([ { text: 'ab', binary: false }, { text: '00FF', binary: true }, - { text: '中cd', binary: false }, + { text: '\u4e2dcd', binary: false }, ]); expect(printableBytes(undefined, '616200ff')).toBe(false); }); @@ -18,8 +18,8 @@ describe('mixed KV byte presentation', () => { expect(displayBytes(undefined, 'e282')).toBe('E282'); }); it('distinguishes text that looks like hex and preserves valid Unicode', () => { - expect(byteRuns('FF00中😀', undefined)).toEqual([{ text: 'FF00中😀', binary: false }]); - expect(printableBytes('FF00中😀', undefined)).toBe(true); + expect(byteRuns('FF00\u4e2d😀', undefined)).toEqual([{ text: 'FF00\u4e2d😀', binary: false }]); + expect(printableBytes('FF00\u4e2d😀', undefined)).toBe(true); expect(byteRuns(undefined, 'efbbbf41')).toEqual([{ text: 'EFBBBF', binary: true }, { text: 'A', binary: false }]); }); }); diff --git a/app/crowdb-web/ui/src/s3/S3View.test.tsx b/app/crowdb-web/ui/src/s3/S3View.test.tsx new file mode 100644 index 000000000..38357052a --- /dev/null +++ b/app/crowdb-web/ui/src/s3/S3View.test.tsx @@ -0,0 +1,72 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +import { act, fireEvent, render, screen, waitFor, within } from '@testing-library/react'; +import { afterEach, expect, it, vi } from 'vitest'; +import { S3View } from '../views/S3View'; +import { SelectionProvider } from '../contexts/SelectionContext'; +import { DomainProvider } from '../contexts/DomainContext'; +import { Domain } from '../types'; + +vi.mock('../contexts/ActivityContext', () => ({ useActivity: () => ({ log: vi.fn() }) })); +vi.mock('./useObjectLocations', () => ({ useObjectLocations: () => ({ selected: null, page: null, snapshot: () => null }) })); +afterEach(() => { vi.unstubAllGlobals(); }); + +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise(complete => { resolve = complete; }); + return { promise, resolve }; +} + +const listing = (keys: string[]) => new Response(`false${keys.map(key => + `${key}12etagtoday`).join('')}`); + +async function deletion() { + const removed = deferred(); + const refreshed = deferred(); + let lists = 0; + const fetcher = vi.fn(async (input: RequestInfo | URL, options?: RequestInit) => { + const path = new URL(String(input), 'http://localhost').pathname; + if (path === '/api/access/connections') return new Response(JSON.stringify({ s3: 'http://s3' })); + if (path === '/api/access/s3/') return new Response('photos'); + if (path === '/api/access/s3/photos') return ++lists === 1 ? listing(['delete.txt', 'keep.txt', 'delete.txt.backup']) : refreshed.promise; + if (path === '/api/access/s3/photos/delete.txt') { + if (options?.method === 'DELETE') return removed.promise; + if (options?.method === 'HEAD') return new Response(null, { headers: { 'content-length': '12' } }); + } + throw new Error(`Unexpected request: ${options?.method} ${input}`); + }); + vi.stubGlobal('fetch', fetcher); + render(); + const buckets = await screen.findByRole('table', { name: 'S3 bucket list' }); + fireEvent.click(await within(buckets).findByRole('button', { name: 'photos' })); + fireEvent.click(await screen.findByRole('button', { name: 'delete.txt' })); + await waitFor(() => expect(screen.getByRole('region', { name: 'Object metadata' })).toHaveTextContent('content length12')); + fireEvent.contextMenu(screen.getByRole('heading', { level: 1, name: 'delete.txt' })); + fireEvent.click(screen.getByRole('menuitem', { name: 'Delete object…' })); + fireEvent.click(within(screen.getByRole('dialog', { name: 'Delete object' })).getByRole('button', { name: 'Delete object' })); + return { removed, refreshed, fetcher, lists: () => lists }; +} + +it('removes only the confirmed object while its list refresh is still pending', async () => { + const state = await deletion(); + await act(async () => state.removed.resolve(new Response(null, { status: 204 }))); + await waitFor(() => expect(state.lists()).toBe(2)); + const table = screen.getByRole('table', { name: 'S3 objects' }); + expect(within(table).queryByRole('button', { name: 'delete.txt' })).toBeNull(); + expect(within(table).getByRole('button', { name: 'keep.txt' })).toBeVisible(); + expect(within(table).getByRole('button', { name: 'delete.txt.backup' })).toBeVisible(); + await act(async () => state.refreshed.resolve(new Response('refresh unavailable', { status: 503 }))); + expect(screen.getByRole('alert')).toHaveTextContent('refresh unavailable'); + expect(within(table).queryByRole('button', { name: 'delete.txt' })).toBeNull(); + expect(state.fetcher.mock.calls.filter(([, options]) => options?.method === 'DELETE')).toHaveLength(1); +}); + +it.each([403, 503])('retains the selected row after a rejected or unknown delete (%s)', async status => { + const state = await deletion(); + await act(async () => state.removed.resolve(new Response('delete unavailable', { status }))); + expect(screen.getByRole('alert')).toHaveTextContent('delete unavailable'); + expect(screen.getByRole('heading', { level: 1, name: 'delete.txt' })).toBeVisible(); + expect(state.lists()).toBe(1); + expect(state.fetcher.mock.calls.filter(([, options]) => options?.method === 'DELETE')).toHaveLength(1); +}); diff --git a/app/crowdb-web/ui/src/services/useNodeServicePlans.test.tsx b/app/crowdb-web/ui/src/services/useNodeServicePlans.test.tsx index b300d5a6b..1f4d56d59 100644 --- a/app/crowdb-web/ui/src/services/useNodeServicePlans.test.tsx +++ b/app/crowdb-web/ui/src/services/useNodeServicePlans.test.tsx @@ -65,10 +65,11 @@ describe('node service plans', () => { existing.push({ node_id: 3, service_type: 'diskio', pid: 20 }); vi.mocked(listServers).mockResolvedValue(existing as Awaited>); let ownershipReady = false; - vi.mocked(serviceRequest).mockImplementation(async path => { + vi.mocked(serviceRequest).mockImplementation(async (path, method) => { if (path === '/service-plans') return { 1: { revision: 7, steps: first }, 2: { revision: 7, steps: second } }; if (path === '/chunk-storage-readiness') return { ready: ownershipReady, reason: ownershipReady ? undefined : 'Waiting: storage' }; if (path === '/deployment-defaults') return { 'access-server': { instance_id: '2', http_port: 9092, s3_port: 9091 } }; + if (path === '/nodes/2/services/deploy' && method === 'POST') existing.push({ node_id: 2, service_type: 'access-server', pid: 31 }); return { revision: 8, ready: true }; }); const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); @@ -78,7 +79,7 @@ describe('node service plans', () => { expect(deployments()).toHaveLength(0); expect(result.current.plans[2]['access-server'].state).toBe('waiting'); ownershipReady = true; - vi.mocked(listServers).mockResolvedValue([...existing, { node_id: 2, service_type: 'chunk-kv', pid: 30 }] as Awaited>); + existing.push({ node_id: 2, service_type: 'chunk-kv', pid: 30 }); await act(async () => { await vi.advanceTimersByTimeAsync(600); }); expect(deployments()).toHaveLength(1); expect(result.current.plans[2]['access-server'].state).toBe('deployed'); @@ -142,15 +143,18 @@ describe('node service plans', () => { for (const kind of serviceOrder) steps[kind] = { state: 'disabled' }; steps['chunk-kv'] = { state: 'waiting' }; let restarted = false; + let chunkKvRunning = false; vi.mocked(listServers).mockImplementation(async () => [ { node_id: 1, service_type: 'chunkdb', pid: 12 }, { node_id: 1, service_type: 'diskio', pid: 13 }, { node_id: 2, service_type: 'diskio', pid: restarted ? 14 : undefined }, + ...(chunkKvRunning ? [{ node_id: 1, service_type: 'chunk-kv', pid: 15 }] : []), ] as Awaited>); const request = vi.mocked(serviceRequest).getMockImplementation()!; vi.mocked(serviceRequest).mockImplementation(async (path, method, body) => { if (path === '/service-plans') return { 1: { revision: 1, steps } }; if (path === '/deployment-defaults') return { 'chunk-kv': { instance_id: '1', http_port: 12010, rpc_port: 12110 } }; + if (path === '/nodes/1/services/deploy' && method === 'POST') chunkKvRunning = true; return request(path, method, body); }); const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); @@ -178,11 +182,13 @@ describe('node service plans', () => { expect(serviceRequest).toHaveBeenCalledWith('/nodes/1/service-plan', 'PUT', expect.objectContaining({ revision: 7 })); }); it('keeps a deployment queued when ownership changes after its readiness probe', async () => { - vi.mocked(listServers).mockResolvedValue([{ node_id: 1, service_type: 'diskio', pid: 10 }] as Awaited>); + const existing = [{ node_id: 1, service_type: 'diskio', pid: 10 }] as Awaited>; + vi.mocked(listServers).mockResolvedValue(existing); let ownershipPublished = false; const request = vi.mocked(serviceRequest).getMockImplementation()!; vi.mocked(serviceRequest).mockImplementation(async (path, method, body) => { if (path.endsWith('/services/deploy') && !ownershipPublished) throw new Error('HTTP 409: Waiting: every configured disk group needs live DiskIO ownership'); + if (path.endsWith('/services/deploy') && method === 'POST') existing.push({ node_id: 1, service_type: 'chunkdb', pid: 11 } as typeof existing[number]); return request(path, method, body); }); const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); @@ -320,20 +326,23 @@ describe('node service plans', () => { expect(result.current.plans[1]['access-server'].state).toBe('deployed'); expect(vi.mocked(serviceRequest).mock.calls.filter(([, method]) => method === 'POST')).toHaveLength(3); }); - it('does not retry failures automatically and stops queued deployments before reset', async () => { + it('observes failures without replaying mutations and stops observation before reset', async () => { vi.mocked(serviceRequest).mockImplementation(async path => { if (path === '/service-plans') return {}; if (path === '/chunk-storage-readiness' || path === '/group0-readiness') return { ready: true }; if (path.endsWith('/service-plan')) return { revision: 1, ready: true }; throw new Error('Unavailable'); }); const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); await act(async () => { result.current.start(1); }); expect(result.current.plans[1].chunkdb.state).toBe('failed'); - const attempts = vi.mocked(serviceRequest).mock.calls.length; + const mutations = vi.mocked(serviceRequest).mock.calls.filter(([, method]) => method === 'POST'); + const observations = vi.mocked(listServers).mock.calls.length; await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); - expect(serviceRequest).toHaveBeenCalledTimes(attempts); + expect(vi.mocked(listServers).mock.calls.length).toBeGreaterThan(observations); + expect(vi.mocked(serviceRequest).mock.calls.filter(([, method]) => method === 'POST')).toEqual(mutations); + const attempts = vi.mocked(serviceRequest).mock.calls.length; await act(async () => { await result.current.stop(); }); await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); expect(result.current.plans).toEqual({}); expect(serviceRequest).toHaveBeenCalledTimes(attempts); }); - it('recovers waiting progress and fences interrupted deployment until explicit reconciliation', async () => { + it('recovers waiting progress and observes an interrupted deployment without replaying it', async () => { const steps = Object.fromEntries(['paxos-kv', 'diskdb', 'chunkdb', 'diskio', 'chunk-kv', 'access-server'].map(kind => [kind, { state: kind === 'chunkdb' ? 'deploying' : kind === 'paxos-kv' || kind === 'diskdb' ? 'deployed' : 'waiting' }])); vi.mocked(serviceRequest).mockImplementation(async path => path === '/service-plans' ? { 1: { revision: 7, steps } } : path === '/chunk-storage-readiness' ? { ready: true } : { revision: 8, ready: true }); @@ -346,7 +355,7 @@ describe('node service plans', () => { { node_id: 1, service_type: 'paxos-kv', pid: 10 }, { node_id: 1, service_type: 'diskdb', pid: 11 }, { node_id: 1, service_type: 'chunkdb', pid: 12 }, ] as Awaited>); - await act(async () => { await result.current.start(1); }); + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); expect(result.current.plans[1].chunkdb.state).toBe('deployed'); expect(vi.mocked(serviceRequest).mock.calls.some(([path, method, body]) => path.endsWith('/services/deploy') && method === 'POST' && (body as { kind?: string })?.kind === 'chunkdb')).toBe(false); expect(serviceRequest).toHaveBeenCalledWith('/nodes/1/service-plan', 'PUT', expect.objectContaining({ revision: 7 })); @@ -414,7 +423,8 @@ describe('node service plans', () => { vi.mocked(listServers).mockRejectedValue(new Error('Registry unavailable')); const { result } = renderHook(() => useNodeServicePlans([], {}, async () => {}, true)); await act(async () => { await result.current.start(1, ['paxos-kv']); }); - expect(result.current.plans[1]['paxos-kv'].state).toBe('failed'); + expect(result.current.plans[1]['paxos-kv'].state).toBe('waiting'); + expect(result.current.plans[1]['paxos-kv'].detail).toContain('inventory'); for (const kind of serviceOrder.filter(kind => kind !== 'paxos-kv')) expect(result.current.plans[1][kind].state).toBe('disabled'); }); it('rechecks registered DiskIO ownership promptly without requiring disk rows to be expanded', async () => { diff --git a/app/crowdb-web/ui/src/services/useNodeServicePlans.ts b/app/crowdb-web/ui/src/services/useNodeServicePlans.ts index 91f88985d..079ea5fc8 100644 --- a/app/crowdb-web/ui/src/services/useNodeServicePlans.ts +++ b/app/crowdb-web/ui/src/services/useNodeServicePlans.ts @@ -11,6 +11,8 @@ export const serviceOrder = ['access-server', 'chunk-kv', 'chunkdb', 'diskdb', ' export type ServiceKind = typeof serviceOrder[number]; const deploymentOrder: ServiceKind[] = ['paxos-kv', 'diskdb', 'diskio', 'chunkdb', 'chunk-kv', 'access-server']; const startingService = 'Waiting: target monitor is starting the registered service'; +const observingService = 'Waiting: current process observation is unavailable'; +const observingInventory = 'Waiting: current service inventory is unavailable'; export const serviceLabels = { 'access-server': 'crowdb-access-server', 'chunk-kv': 'crowdb-chunk-kv', @@ -24,8 +26,27 @@ export type NodeServicePlan = Record; export type ServiceOverrides = Partial> & { keepalive_interval_secs?: number }>>; export const newPlan = (): NodeServicePlan => Object.fromEntries(serviceOrder.map(kind => [kind, { state: 'pending' }])) as NodeServicePlan; +function serviceFailureDetail(kind: ServiceKind, overrides: ServiceOverrides | undefined, error: unknown) { + const listeners = Object.entries(overrides?.[kind] ?? {}).map(([name, port]) => `${name}=${port}`).join(', '); + return `${serviceLabels[kind]}${listeners ? ` (${listeners})` : ''}: ${String(error)}`.slice(0, 4096); +} + +function observationDelay(plans: Record, hasDisks: boolean, waitingForStorage: boolean, immediate: boolean) { + if (immediate) return 0; + const steps = Object.values(plans).flatMap(plan => serviceOrder.map(kind => plan[kind])); + const queued = steps.some(step => step.state === 'waiting'); + const observing = steps.some(step => step.state === 'failed' + || step.detail === startingService || step.detail === observingService + || step.detail?.startsWith(observingInventory) + || step.detail?.startsWith('Waiting: restart registered DiskIO services')); + const waitingForAuthority = steps.some(step => step.detail === 'Waiting: initialize Group 0 in Paxos KV'); + // Missing Capacity data does not delay observations of registered services. + return queued && !hasDisks && !observing && !waitingForAuthority && !waitingForStorage ? 10000 : 2000; +} + /** Serializes default deployments across nodes so each step gets fresh port reservations. - * Plans outlive dialogs; dependency changes resume waiting steps, failures need Retry. + * Plans outlive dialogs; dependency changes and observed recovery update progress. + * Failed lifecycle mutations need explicit Retry; observation never replays them. * Progress is durable; interrupted mutations require registration reconciliation. */ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record, refresh: () => Promise, enabled: boolean) { @@ -75,7 +96,7 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< ? { ...savedStep, state: 'warning' as const } : savedStep; return [kind, normalizedStep.state === 'deploying' - ? { state: 'failed', detail: 'Deployment was interrupted; verify the registered service, then Retry to reconcile.' } : normalizedStep]; + ? { state: 'failed', detail: 'Deployment was interrupted; checking the registered service. Retry only if it still needs to be started.' } : normalizedStep]; })) as NodeServicePlan; update(id, steps); } @@ -132,7 +153,7 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< if (disposed || stopped.current) return; for (const [key, initial] of Object.entries(plansRef.current)) { if (disposed || stopped.current) break; - if (!serviceOrder.some(kind => ['pending', 'waiting'].includes(initial[kind].state))) continue; + if (!serviceOrder.some(kind => ['pending', 'waiting', 'failed', 'deployed'].includes(initial[kind].state))) continue; const id = Number(key); let plan = plansRef.current[id]; const write = async (kind: ServiceKind, step: ServiceStep) => { plan = { ...plan, [kind]: step }; await persist(id, plan); }; @@ -140,7 +161,9 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< try { existing = await listServers(); } catch (error) { if (disposed || stopped.current) return; - for (const kind of serviceOrder) if (!['deployed', 'disabled'].includes(plan[kind].state)) await write(kind, { state: 'failed', detail: String(error) }); + for (const kind of serviceOrder) if (['pending', 'waiting'].includes(plan[kind].state)) { + await write(kind, { state: 'waiting', detail: `${observingInventory}: ${String(error)}` }); + } continue; } let authorityReady = false; @@ -152,8 +175,12 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< for (const kind of deploymentOrder) { if (disposed || stopped.current || !input.current.enabled) break; if (plan[kind].state === 'disabled') continue; - if (plan[kind].state === 'failed' || plan[kind].state === 'warning') continue; + if (plan[kind].state === 'warning') continue; const registered = existing.find(server => server.node_id === id && server.service_type === kind); + if (plan[kind].state === 'failed') { + if (registered?.pid && registered.process_observed !== false) await write(kind, { state: 'deployed' }); + continue; + } if (registered) { const retryKey = `${id}/${kind}`; if (!registered.pid && retryRegistered.current.has(retryKey)) { @@ -174,8 +201,10 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< retryRegistered.current.delete(retryKey); const step: ServiceStep = registered.pid ? { state: 'deployed' } : registered.start_pending ? { state: 'waiting', detail: startingService } - : { state: 'failed', detail: 'Registered service is stopped; restart it from the Node menu before resuming.' }; - if (plan[kind].state !== step.state) await write(kind, step); + : registered.process_observed === false + ? { state: 'waiting', detail: observingService } + : { state: 'failed', detail: serviceFailureDetail(kind, overrides.current[id], 'Registered service is stopped; restart it from the Node menu before resuming.') }; + if (plan[kind].state !== step.state || plan[kind].detail !== step.detail) await write(kind, step); continue; } retryRegistered.current.delete(`${id}/${kind}`); @@ -225,16 +254,15 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< deploymentStarted = true; } catch (error) { if (disposed || stopped.current) return; - // Hardware can change between the readiness probe and mutation. - // A rejected prerequisite has not launched a service; keep it queued. - const dependency = String(error).match(/^Error: HTTP 409: (Waiting: .+)$/); - if (dependency && (kind === 'chunkdb' || kind === 'chunk-kv')) { + // A prerequisite can change after its probe. The monitor may + // already own the intent; the next pass observes that registration. + const dependency = String(error).match(/^Error: HTTP 409: (Waiting: [\s\S]+)$/); + if (dependency && (kind === 'chunkdb' || kind === 'chunk-kv' || kind === 'access-server')) { waitingForStorage = true; await write(kind, { state: 'waiting', detail: dependency[1] }); continue; } - const listeners = Object.entries(overrides.current[id]?.[kind] ?? {}).map(([name, port]) => `${name}=${port}`).join(', '); - const detail = `${serviceLabels[kind]}${listeners ? ` (${listeners})` : ''}: ${String(error)}`.slice(0, 4096); + const detail = serviceFailureDetail(kind, overrides.current[id], error); // A node without a fixed ChunkDB slot is valid. Keep it as a // warning so the rest of the node plan and existing balance // services continue running. @@ -250,13 +278,8 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< } finally { busy.current = false; } } if (!disposed) { - const hasQueuedWork = Object.values(plansRef.current).some(plan => serviceOrder.some(kind => plan[kind].state === 'waiting')); - const waitingForAuthority = Object.values(plansRef.current).some(plan => serviceOrder.some(kind => plan[kind].detail === 'Waiting: initialize Group 0 in Paxos KV')); - const waitingForRestart = Object.values(plansRef.current).some(plan => serviceOrder.some(kind => plan[kind].detail?.startsWith('Waiting: restart registered DiskIO services'))); - const waitingForStart = Object.values(plansRef.current).some(plan => serviceOrder.some(kind => plan[kind].detail === startingService)); const hasDisks = Object.values(input.current.groups).some(node => Object.values(node.disksByDg).some(disks => disks.length > 0)); - // Collapsed disk rows are not evidence that registered storage is absent. - const delay = wakePending.current ? 0 : hasQueuedWork && !hasDisks && !waitingForAuthority && !waitingForRestart && !waitingForStart && !waitingForStorage ? 10000 : 2000; + const delay = observationDelay(plansRef.current, hasDisks, waitingForStorage, wakePending.current); wakePending.current = false; timer = setTimeout(() => { running.current = tick(); }, delay); } diff --git a/app/crowdb-web/ui/src/services/useNodeServicePlansDependencies.test.tsx b/app/crowdb-web/ui/src/services/useNodeServicePlansDependencies.test.tsx new file mode 100644 index 000000000..4f8bb2ff9 --- /dev/null +++ b/app/crowdb-web/ui/src/services/useNodeServicePlansDependencies.test.tsx @@ -0,0 +1,54 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +import { act, renderHook } from '@testing-library/react'; +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { listServers, restartServer, restartDiskdb } from '../api'; +import { serviceRequest } from './client'; +import { useNodeServicePlans } from './useNodeServicePlans'; +import type { EnrichedStoreView } from '../types'; + +vi.mock('../api', () => ({ listServers: vi.fn(), deployServer: vi.fn(), deployDiskdb: vi.fn(), restartServer: vi.fn(), restartDiskdb: vi.fn() })); +vi.mock('./client', () => ({ serviceNames: {}, serviceRequest: vi.fn() })); + +beforeEach(() => { vi.useFakeTimers(); }); +afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); }); + +it.each([ + ['chunk-kv', 'Waiting: config error: monitor SSH command failed: Error: managed service exited before readiness: node=1 service=chunk-kv-1 exit_code=1\n'], + ['access-server', 'Waiting: service dependency has no committed intent'], +] as const)('observes the accepted %s dependency wait without another deployment or restart', async (kind, detail) => { + const stores = [{ store_id: '0', groups: [{ group_id: '0' }, { group_id: '1' }] }] as unknown as EnrichedStoreView[]; + let accepted = false; + let running = false; + vi.mocked(listServers).mockImplementation(async () => [ + { node_id: 1, service_type: 'chunkdb', pid: 11 }, + { node_id: 1, service_type: 'diskio', pid: 12 }, + { node_id: 2, service_type: 'diskio', pid: 22 }, + { node_id: 2, service_type: 'chunk-kv', pid: 23 }, + ...(accepted ? [{ id: `${kind}-1`, node_id: 1, service_type: kind, + pid: running ? 42 : undefined, start_pending: !running }] : []), + ] as Awaited>); + vi.mocked(serviceRequest).mockImplementation(async (path, method) => { + if (path === '/service-plans') return {}; + if (path === '/deployment-defaults') return { [kind]: { instance_id: '1', http_port: 7400, rpc_port: 7401 } }; + if (path === '/nodes/1/services/deploy' && method === 'POST') { + accepted = true; + throw new Error(`HTTP 409: ${detail}`); + } + return { revision: 1, ready: true }; + }); + const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); + await act(async () => { await result.current.start(1, [kind]); }); + expect(result.current.plans[1][kind]).toEqual({ state: 'waiting', detail }); + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1][kind]).toEqual({ state: 'waiting', detail: 'Waiting: target monitor is starting the registered service' }); + running = true; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1][kind]).toEqual({ state: 'deployed' }); + expect(vi.mocked(serviceRequest).mock.calls.filter(([path, method]) => + path.endsWith('/services/deploy') && method === 'POST')).toHaveLength(1); + expect(restartServer).not.toHaveBeenCalled(); + expect(restartDiskdb).not.toHaveBeenCalled(); + expect(vi.mocked(serviceRequest).mock.calls.some(([path]) => path.endsWith('/restart'))).toBe(false); +}); diff --git a/app/crowdb-web/ui/src/services/useNodeServicePlansObservation.test.tsx b/app/crowdb-web/ui/src/services/useNodeServicePlansObservation.test.tsx new file mode 100644 index 000000000..61284d522 --- /dev/null +++ b/app/crowdb-web/ui/src/services/useNodeServicePlansObservation.test.tsx @@ -0,0 +1,270 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. +import { act, renderHook } from '@testing-library/react'; +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { deployServer, deployDiskdb, listServers, restartServer, restartDiskdb } from '../api'; +import type { ServerSummary } from '../api'; +import { serviceRequest } from './client'; +import { newPlan, serviceOrder, useNodeServicePlans } from './useNodeServicePlans'; +import type { EnrichedStoreView } from '../types'; + +vi.mock('../api', () => ({ listServers: vi.fn(), deployServer: vi.fn(), deployDiskdb: vi.fn(), restartServer: vi.fn(), restartDiskdb: vi.fn() })); +vi.mock('./client', () => ({ serviceNames: {}, serviceRequest: vi.fn() })); +type ProcessState = 'running' | 'stopped' | 'unknown'; +type ObservedServer = ServerSummary & { process_observed: boolean }; +const stores = [{ store_id: '0', groups: [{ group_id: '0' }, { group_id: '1' }] }] as unknown as EnrichedStoreView[]; +const kv = (state: ProcessState): ObservedServer => ({ id: 'kv-1', node_id: 1, service_type: 'paxos-kv', + pid: state === 'running' ? 126 : undefined, process_observed: state !== 'unknown', start_pending: false, health: 'up' }); +const diskdb = (running: boolean, pending: boolean): ObservedServer => ({ id: 'diskdb-1', node_id: 1, + service_type: 'diskdb', pid: running ? 272 : undefined, process_observed: !pending, + start_pending: pending, health: 'up' }); +function savedPlan() { + const steps = newPlan(); + for (const kind of serviceOrder) steps[kind] = { state: 'disabled' }; + steps['paxos-kv'] = { state: 'deployed' }; + steps.diskdb = { state: 'waiting', detail: 'Waiting: target monitor is starting the registered service' }; + vi.mocked(serviceRequest).mockImplementation(async path => path === '/service-plans' + ? { 1: { revision: 7, steps } } : { revision: 8, ready: true }); +} +function noLifecycleMutation() { + expect(deployServer).not.toHaveBeenCalled(); + expect(deployDiskdb).not.toHaveBeenCalled(); + expect(restartServer).not.toHaveBeenCalled(); + expect(restartDiskdb).not.toHaveBeenCalled(); + expect(vi.mocked(serviceRequest).mock.calls.some(([path, method]) => method === 'POST' + && (path.endsWith('/deploy') || path.endsWith('/restart')))).toBe(false); +} +beforeEach(() => { vi.useFakeTimers(); savedPlan(); }); +afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); }); + +it('observes an unknown current PKV process fact and recovers the same PID without Retry', async () => { + let state: ProcessState = 'running'; + let finished = false; + vi.mocked(listServers).mockImplementation(async () => [kv(state), diskdb(finished, !finished)]); + const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); + await act(async () => {}); + expect(result.current.plans[1]['paxos-kv'].state).toBe('deployed'); + state = 'unknown'; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('waiting'); + expect(result.current.plans[1]['paxos-kv'].detail).toMatch(/process|observation/i); + noLifecycleMutation(); + state = 'running'; finished = true; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('deployed'); + expect(result.current.plans[1].diskdb.state).toBe('deployed'); + noLifecycleMutation(); +}); + +it.each([['confirmed observation', true], ['legacy field omitted', undefined]] as const)('observes a stopped PKV and reconciles its recovery without a lifecycle mutation with %s', async (_label, processObserved) => { + let state: ProcessState = 'running'; + vi.mocked(listServers).mockImplementation(async () => [{ ...kv(state), process_observed: processObserved }, diskdb(false, true)]); + const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); + await act(async () => {}); + expect(result.current.plans[1]['paxos-kv'].state).toBe('deployed'); + state = 'stopped'; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('failed'); + await act(async () => { await vi.advanceTimersByTimeAsync(4000); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('failed'); + noLifecycleMutation(); + state = 'running'; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('deployed'); + noLifecycleMutation(); +}); + +it('observes an exhausted startup without restarting it and reconciles a later owned PID', async () => { + let exhausted = false; + let laterPid = false; + vi.mocked(listServers).mockImplementation(async () => [kv('running'), diskdb(laterPid, !exhausted)]); + const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); + await act(async () => {}); + expect(result.current.plans[1].diskdb.state).toBe('waiting'); + exhausted = true; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1].diskdb.state).toBe('failed'); + await act(async () => { await vi.advanceTimersByTimeAsync(4000); }); + expect(result.current.plans[1].diskdb.state).toBe('failed'); + noLifecycleMutation(); + laterPid = true; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1].diskdb.state).toBe('deployed'); + noLifecycleMutation(); +}); + +it('observes unknown process state every two seconds without loaded Capacity data', async () => { + let state: ProcessState = 'running'; + let finished = false; + vi.mocked(listServers).mockImplementation(async () => [kv(state), diskdb(finished, !finished)]); + const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); + await act(async () => {}); + expect(result.current.plans[1]['paxos-kv'].state).toBe('deployed'); + // Finish the only other queued service before observing the unknown process. + // The next timer must observe this process rather than use the disk backoff. + state = 'unknown'; finished = true; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('waiting'); + expect(result.current.plans[1].diskdb.state).toBe('deployed'); + state = 'running'; + await act(async () => { await vi.advanceTimersByTimeAsync(1999); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('waiting'); + await act(async () => { await vi.advanceTimersByTimeAsync(1); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('deployed'); + noLifecycleMutation(); +}); + +function failedPlan() { + const steps = newPlan(); + for (const kind of serviceOrder) steps[kind] = { state: 'disabled' }; + steps['paxos-kv'] = { state: 'failed', detail: 'Previously reported stopped' }; + return steps; +} +function recoverPlan(steps = failedPlan(), second?: ReturnType) { + vi.mocked(serviceRequest).mockImplementation(async path => path === '/service-plans' + ? { 1: { revision: 7, steps }, ...(second ? { 2: { revision: 3, steps: second } } : {}) } + : { revision: 8, ready: true }); +} + +it('observes a persisted failed-only plan and recovers only after current process evidence returns', async () => { + recoverPlan(); + let state: ProcessState = 'unknown'; + vi.mocked(listServers).mockImplementation(async () => [kv(state)]); + const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); + await act(async () => {}); + expect(result.current.plans[1]['paxos-kv'].state).toBe('failed'); + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('failed'); + noLifecycleMutation(); + state = 'running'; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('deployed'); + noLifecycleMutation(); +}); + +it('does not deploy a failed missing service while observing its later registered recovery', async () => { + recoverPlan(); + let present = false; + vi.mocked(listServers).mockImplementation(async () => present ? [kv('running')] : []); + const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); + await act(async () => { await vi.advanceTimersByTimeAsync(4000); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('failed'); + noLifecycleMutation(); + present = true; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('deployed'); + noLifecycleMutation(); +}); + +it('keeps failed-service observation at two seconds while another plan waits for unloaded disks', async () => { + const second = failedPlan(); + second['paxos-kv'] = { state: 'disabled' }; + second['chunk-kv'] = { state: 'waiting', detail: 'Waiting: deploy DiskIO on at least two nodes for journal mirrors' }; + recoverPlan(failedPlan(), second); + let state: ProcessState = 'stopped'; + vi.mocked(listServers).mockImplementation(async () => [kv(state)]); + const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); + await act(async () => {}); + await act(async () => { await vi.advanceTimersByTimeAsync(0); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('failed'); + state = 'running'; + await act(async () => { await vi.advanceTimersByTimeAsync(1999); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('failed'); + await act(async () => { await vi.advanceTimersByTimeAsync(1); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('deployed'); + noLifecycleMutation(); +}); + + +it('preserves an existing failure and fixed-slot warning across inventory read errors and observes recovery', async () => { + const steps = failedPlan(); + steps.chunkdb = { state: 'warning', detail: 'Healthy service outside the fixed slot plan' }; + recoverPlan(steps); + let unavailable = true; + vi.mocked(listServers).mockImplementation(async () => { + if (unavailable) throw new Error('Controlled inventory unavailable'); + return [kv('running')]; + }); + const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1]['paxos-kv']).toEqual(steps['paxos-kv']); + expect(result.current.plans[1].chunkdb).toEqual(steps.chunkdb); + noLifecycleMutation(); + unavailable = false; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('deployed'); + expect(result.current.plans[1].chunkdb).toEqual(steps.chunkdb); + noLifecycleMutation(); +}); + +it('keeps unattempted work waiting across an unavailable inventory and resumes observation within two seconds', async () => { + const steps = failedPlan(); + steps['paxos-kv'] = { state: 'pending' }; + recoverPlan(steps); + let unavailable = true; + vi.mocked(listServers).mockImplementation(async () => { + if (unavailable) throw new Error('Controlled inventory unavailable'); + return [kv('running')]; + }); + const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); + await act(async () => { await vi.advanceTimersByTimeAsync(0); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('waiting'); + expect(result.current.plans[1]['paxos-kv'].detail).toContain('inventory'); + noLifecycleMutation(); + unavailable = false; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('deployed'); + noLifecycleMutation(); +}); + +it.each(['stopped', 'unknown'] as const)('continues observing a completed plan after a transient PID becomes %s', async firstLoss => { + const steps = failedPlan(); + steps['paxos-kv'] = { state: 'deployed' }; + steps.diskdb = { state: 'failed', detail: 'crowdb-disk-db (rpc_port=7100): Previously reported stopped' }; + const overrides = { diskdb: { rpc_port: 7100 } }; + vi.mocked(serviceRequest).mockImplementation(async path => path === '/service-plans' + ? { 1: { revision: 7, steps, overrides } } : { revision: 8, ready: true }); + let state: ProcessState = 'unknown'; + vi.mocked(listServers).mockImplementation(async () => [kv('running'), { + ...diskdb(state === 'running', false), process_observed: state !== 'unknown', + }]); + const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); + const preserved = () => { + expect(result.current.plans[1]['paxos-kv']).toEqual(steps['paxos-kv']); + for (const kind of serviceOrder) if (kind !== 'paxos-kv' && kind !== 'diskdb') { + expect(result.current.plans[1][kind]).toEqual({ state: 'disabled' }); + } + const writes = vi.mocked(serviceRequest).mock.calls.filter(([path, method]) => path === '/nodes/1/service-plan' && method === 'PUT'); + for (const [, , body] of writes) expect(body).toMatchObject({ overrides }); + noLifecycleMutation(); + }; + await act(async () => {}); + expect(result.current.plans[1].diskdb).toEqual(steps.diskdb); + preserved(); + state = 'running'; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1].diskdb.state).toBe('deployed'); + expect(serviceOrder.every(kind => ['deployed', 'disabled'].includes(result.current.plans[1][kind].state))).toBe(true); + preserved(); + state = firstLoss; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1].diskdb.state).toBe(firstLoss === 'unknown' ? 'waiting' : 'failed'); + preserved(); + if (firstLoss === 'unknown') { + expect(result.current.plans[1].diskdb.detail).toMatch(/process|observation/i); + state = 'stopped'; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1].diskdb.state).toBe('failed'); + } + expect(result.current.plans[1].diskdb.detail).toContain('crowdb-disk-db'); + expect(result.current.plans[1].diskdb.detail).toContain('7100'); + preserved(); + state = 'unknown'; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1].diskdb.state).toBe('failed'); + preserved(); + state = 'running'; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1].diskdb.state).toBe('deployed'); + preserved(); +}); diff --git a/app/crowdb-web/ui/src/views/S3View.tsx b/app/crowdb-web/ui/src/views/S3View.tsx index dacc02031..db0c6f2b7 100644 --- a/app/crowdb-web/ui/src/views/S3View.tsx +++ b/app/crowdb-web/ui/src/views/S3View.tsx @@ -169,6 +169,7 @@ export function S3View({ active, readonly, onChunk }: { active: boolean; readonl if (action === 'abort') { await listUploads(undefined, target.bucket); return; } if (action === 'create' || action === 'delete-bucket') { if (target.bucket === bucket) root(); await listBuckets(undefined, bucketListedPrefix); } else if (target.bucket === bucket) { + if (action === 'delete-object') setRows(current => current.filter(row => row.key !== target.key)); if (selected?.key === target.key) { setPreview(''); setDetail(null); if (action === 'delete-object') setSelected(null); } await loadObjects(bucket, undefined, listedPrefix); if (action === 'replace' && selected && selected.key === target.key) { diff --git a/container/crowdb-monitor/src/bootstrap/iceberg.rs b/container/crowdb-monitor/src/bootstrap/iceberg.rs index b35f025e0..0a16f4b60 100644 --- a/container/crowdb-monitor/src/bootstrap/iceberg.rs +++ b/container/crowdb-monitor/src/bootstrap/iceberg.rs @@ -11,6 +11,7 @@ use crate::{ MonitorLogError, ServerCredentials, }; +mod command_failure; mod console; mod shared_console; @@ -28,6 +29,12 @@ pub enum IcebergBootstrapError { Conflict(&'static str), #[error("Iceberg management command failed: {0}")] Command(&'static str), + #[error("Iceberg management command {action} failed ({status}): {diagnostic}")] + CommandExit { + action: String, + status: String, + diagnostic: String, + }, #[error("Iceberg management process failed: {0}")] Io(#[from] std::io::Error), #[error("Iceberg bootstrap manifest failed: {0}")] @@ -319,9 +326,7 @@ impl ManagementCommand<'_> { .await .map_err(|_| IcebergBootstrapError::Command("management command timed out"))??; if !output.status.success() { - return Err(IcebergBootstrapError::Command( - "management command exited unsuccessfully", - )); + return Err(command_failure::describe(arguments, &output, self.credentials)); } Ok(output.stdout) } diff --git a/container/crowdb-monitor/src/bootstrap/iceberg/command_failure.rs b/container/crowdb-monitor/src/bootstrap/iceberg/command_failure.rs new file mode 100644 index 000000000..8e4cdf4b3 --- /dev/null +++ b/container/crowdb-monitor/src/bootstrap/iceberg/command_failure.rs @@ -0,0 +1,36 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::IcebergBootstrapError; +use crate::ServerCredentials; +use std::process::Output; + +pub(super) fn describe( + arguments: &[&str], + output: &Output, + credentials: &ServerCredentials, +) -> IcebergBootstrapError { + let mut diagnostic = String::from_utf8_lossy(&output.stderr).into_owned(); + // Redact before truncation so a boundary cannot expose part of a credential. + for line in credentials.server_env().lines() { + if let Some((_, value)) = line.split_once('=') { + if !value.is_empty() { + diagnostic = diagnostic.replace(value, ""); + } + } + } + let diagnostic: String = diagnostic + .chars() + .take(1024) + .map(|character| if character.is_control() { ' ' } else { character }) + .collect(); + IcebergBootstrapError::CommandExit { + action: arguments.first().copied().unwrap_or("unknown").to_owned(), + status: output.status.to_string(), + diagnostic: if diagnostic.trim().is_empty() { + "no stderr was produced".into() + } else { + diagnostic + }, + } +} diff --git a/container/crowdb-monitor/src/lib.rs b/container/crowdb-monitor/src/lib.rs index e4f72233e..aa10172fd 100644 --- a/container/crowdb-monitor/src/lib.rs +++ b/container/crowdb-monitor/src/lib.rs @@ -31,8 +31,6 @@ pub use liveness::{probe_liveness, LivenessError, LivenessServer}; pub use manifest::{BootstrapManifest, BootstrapSession, ManifestError, ManifestState}; pub use monitor_log::{MonitorEvent, MonitorEventKind, MonitorLog, MonitorLogError}; #[cfg(feature = "test-util")] -pub use node::unique_member_endpoints_for_tests; -#[cfg(feature = "test-util")] pub use node::{ accept_file_for_tests, accept_member_trust_for_tests, accept_startup_mode_for_tests, check_member_trust_for_tests, check_retained_medium_for_tests, file_operations_for_tests, @@ -42,9 +40,15 @@ pub use node::{ NodeServiceForTests, NodeSetupForTests, SuppliedOperationsForTests, }; pub use node::{ - accept_node_policy, control_node, discover_disks, read_node_policy, run_node, run_node_with_inputs, - serve_node_management, CandidateCache, DiscoveryConfig, DiscoveryError, NodeDiscovery, NodeIdentity, - NodeIdentityError, NodeManagementError, NodeRuntimeConfig, NODE_SERVICE_TYPE, + accept_node_policy, accept_node_policy_with_port_offset, control_node, discover_disks, read_node_policy, + resolve_node_policy_with_port_offset, run_node, run_node_with_inputs, serve_node_management, + CandidateCache, DiscoveryConfig, DiscoveryError, NodeDiscovery, NodeIdentity, NodeIdentityError, + NodeManagementError, NodeRuntimeConfig, NODE_SERVICE_TYPE, +}; +#[cfg(feature = "test-util")] +pub use node::{ + member_recovery_destinations_for_tests, recovery_authority_work_for_tests, + unique_member_endpoints_for_tests, }; #[cfg(feature = "test-util")] pub use preview::{preview_step_names_for_tests, run_preview_for_tests, PreviewError}; @@ -59,7 +63,7 @@ pub use status::{MonitorPhase, MonitorStatus, ServiceStatus, StatusError, Status pub use supervisor::{Supervisor, SupervisorError}; #[cfg(feature = "test-util")] -pub use node::rebind_recovery_services_for_tests; +pub use node::{apply_member_endpoints_for_tests, rebind_recovery_services_for_tests}; pub use node::read_recovery_identity; diff --git a/container/crowdb-monitor/src/monitor_log.rs b/container/crowdb-monitor/src/monitor_log.rs index 70b00dd5f..e8bf10800 100644 --- a/container/crowdb-monitor/src/monitor_log.rs +++ b/container/crowdb-monitor/src/monitor_log.rs @@ -125,4 +125,33 @@ impl MonitorLog { } Ok(()) } + /// Additive diagnostic record; legacy lifecycle event fields stay unchanged. + pub(crate) async fn record_process_exit( + &mut self, + service: &str, + pid: u32, + observation: &serde_json::Value, + ) -> Result<(), MonitorLogError> { + let timestamp_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| MonitorLogError::Clock)? + .as_millis(); + let warning = observation.get("success").and_then(serde_json::Value::as_bool) != Some(true); + let mut body = serde_json::to_vec(&serde_json::json!({ + "timestamp_ms": timestamp_ms, + "level": if warning { "warn" } else { "info" }, + "kind": "child_exit_observed", + "service": service, + "pid": pid, + "attempt": null, + "process_exit": observation, + }))?; + body.push(b'\n'); + self.output.write_record(&body).await?; + self.output.sync().await?; + if warning && self.mirror_warnings_to_stderr { + eprint!("{}", String::from_utf8_lossy(&body)); + } + Ok(()) + } } diff --git a/container/crowdb-monitor/src/node.rs b/container/crowdb-monitor/src/node.rs index 44bdd6239..3ea4de8b3 100644 --- a/container/crowdb-monitor/src/node.rs +++ b/container/crowdb-monitor/src/node.rs @@ -31,8 +31,6 @@ pub use runtime::prepare_supplied_disk_for_tests; #[cfg(feature = "test-util")] pub use runtime::runtime_observation_for_tests; #[cfg(feature = "test-util")] -pub use runtime::unique_member_endpoints_for_tests; -#[cfg(feature = "test-util")] pub use runtime::InitializationForTests; #[cfg(feature = "test-util")] pub use runtime::NodeSetupForTests; @@ -42,10 +40,18 @@ pub use runtime::{ }; pub use runtime::{control_node, run_node, run_node_with_inputs, NodeRuntimeConfig}; #[cfg(feature = "test-util")] +pub use runtime::{ + member_recovery_destinations_for_tests, recovery_authority_work_for_tests, + unique_member_endpoints_for_tests, +}; +#[cfg(feature = "test-util")] pub use runtime::{publish_file_for_tests, resume_file_operations_for_tests}; #[cfg(feature = "test-util")] pub use startup::accept_startup_mode_for_tests; -pub use startup::{accept_node_policy, read_node_policy}; +pub use startup::{ + accept_node_policy, accept_node_policy_with_port_offset, read_node_policy, + resolve_node_policy_with_port_offset, +}; pub use management::{serve_node_management, NodeManagementError}; @@ -66,7 +72,7 @@ pub(crate) fn write_single_binding( pub use runtime::NodeServiceForTests; #[cfg(feature = "test-util")] -pub use runtime::rebind_recovery_services_for_tests; +pub use runtime::{apply_member_endpoints_for_tests, rebind_recovery_services_for_tests}; pub use runtime::read_recovery_identity; diff --git a/container/crowdb-monitor/src/node/runtime.rs b/container/crowdb-monitor/src/node/runtime.rs index 11148c244..ea54bcfe8 100644 --- a/container/crowdb-monitor/src/node/runtime.rs +++ b/container/crowdb-monitor/src/node/runtime.rs @@ -58,11 +58,14 @@ mod recovery; pub use ipc::control_node; use ipc::read_request; #[cfg(feature = "test-util")] -pub use recovery::unique_member_endpoints_for_tests; -#[cfg(feature = "test-util")] pub use recovery::{ accept_member_trust_for_tests, check_member_trust_for_tests, member_trust_ready_for_tests, }; +#[cfg(feature = "test-util")] +pub use recovery::{ + member_recovery_destinations_for_tests, recovery_authority_work_for_tests, + unique_member_endpoints_for_tests, +}; type Result = std::result::Result>; @@ -128,7 +131,13 @@ pub async fn run_node_with_inputs( let web = web_service(&profile, config.bind.port())?; processes.start(&web, &BTreeMap::new()).await?; recovery::publish_local(&root, &config.discovery.addresses)?; - let endpoint_recovery = recovery::start(root.clone(), config.bind.port()); + let web_port = profile + .public_endpoints + .iter() + .find(|endpoint| endpoint.id == "web") + .ok_or("web public endpoint is absent from node profile")? + .port; + let endpoint_recovery = recovery::start(root.clone(), config.bind.port(), web_port); kv::resume(&profile, &mut processes).await?; let (stop, stopped) = tokio::sync::oneshot::channel(); let discovery_root = root.clone(); @@ -186,6 +195,7 @@ pub async fn run_node_with_inputs( } Ok::<(), Box>(()) }.await; + health.stop().await; drop(endpoint_recovery); initialization.stop().await; setup.stop().await; @@ -297,11 +307,17 @@ fn service(profile: &DeploymentProfile, id: &str) -> Result { fn web_service(profile: &DeploymentProfile, monitor_port: u16) -> Result { let root = &profile.paths.data_root; let mut web = service(profile, "web")?; + let web_port = profile + .public_endpoints + .iter() + .find(|endpoint| endpoint.id == "web") + .map(|endpoint| endpoint.port) + .ok_or("web public endpoint is absent from node profile")?; web.args = vec![ "--bind".into(), "0.0.0.0".into(), "--port".into(), - "9090".into(), + web_port.to_string(), "--runtime-dir".into(), root.join("console").to_string_lossy().into_owned(), "--node-monitor".into(), @@ -336,7 +352,7 @@ fn disk_inventory(root: &std::path::Path) -> Result { pub use services::NodeServiceForTests; #[cfg(feature = "test-util")] -pub use recovery::rebind_recovery_services_for_tests; +pub use recovery::{apply_member_endpoints_for_tests, rebind_recovery_services_for_tests}; pub use recovery::read_recovery_identity; diff --git a/container/crowdb-monitor/src/node/runtime/cleanup.rs b/container/crowdb-monitor/src/node/runtime/cleanup.rs index 4af24c9da..65fce0673 100644 --- a/container/crowdb-monitor/src/node/runtime/cleanup.rs +++ b/container/crowdb-monitor/src/node/runtime/cleanup.rs @@ -47,8 +47,9 @@ pub(super) async fn execute( .no_proxy() .timeout(Duration::from_secs(30)) .build()?; + let (management_port, _) = super::kv::ports(profile)?; let response = client - .post("http://127.0.0.1:10000/system/cleanup") + .post(format!("http://127.0.0.1:{management_port}/system/cleanup")) .json(&json!({"bootstrap": bootstrap, "confirm_delete_system_store": true})) .send() .await; diff --git a/container/crowdb-monitor/src/node/runtime/control.rs b/container/crowdb-monitor/src/node/runtime/control.rs index baa6b7a79..a6ad0d5bb 100644 --- a/container/crowdb-monitor/src/node/runtime/control.rs +++ b/container/crowdb-monitor/src/node/runtime/control.rs @@ -61,8 +61,18 @@ pub(super) async fn execute( node_id, admission, } => admission::cancel(profile, processes, bootstrap, node_id, admission).await, - request @ (NodeControl::Identity - | NodeControl::InstallKey { .. } + NodeControl::Identity => { + let mut value = identity(root)?; + let port = profile + .public_endpoints + .iter() + .find(|endpoint| endpoint.id == "web") + .ok_or("web public endpoint is absent from node profile")? + .port; + value["web_port"] = port.into(); + Ok(value) + } + request @ (NodeControl::InstallKey { .. } | NodeControl::RemoveKey { .. } | NodeControl::TrustHost { .. }) => keys(root, request).await, NodeControl::Bind { binding } => kv::bind(profile, processes, binding).await, @@ -137,7 +147,8 @@ async fn bootstrap( let accepted = preparation::accept(root, node_id, bootstrap, manifest, credentials, admission)?; super::recovery::prepare_admitted(root, &accepted).await?; start_kv(profile, processes, &accepted).await?; - Ok(json!({"node_id": node_id, "management_port": 10000, "rpc_port": 10100})) + let (management_port, rpc_port) = kv::ports(profile)?; + Ok(json!({"node_id": node_id, "management_port": management_port, "rpc_port": rpc_port})) } _ => Err("unsupported bootstrap control request".into()), } diff --git a/container/crowdb-monitor/src/node/runtime/health.rs b/container/crowdb-monitor/src/node/runtime/health.rs index f5377daf9..6d9e14c53 100644 --- a/container/crowdb-monitor/src/node/runtime/health.rs +++ b/container/crowdb-monitor/src/node/runtime/health.rs @@ -5,16 +5,18 @@ use std::collections::BTreeMap; use std::time::Duration; use super::{durable, AcceptedNode, Result}; -use crate::{ - DeploymentProfile, MonitorPhase, MonitorStatus, ProcessManager, ServiceProfile, ServiceStatus, - StatusStore, -}; +use crate::{DeploymentProfile, MonitorPhase, MonitorStatus, ProcessManager, ServiceProfile, StatusStore}; + +mod intents; +mod observations; pub(super) struct NodeHealth { store: StatusStore, status: MonitorStatus, attempts: BTreeMap, + probe_failures: BTreeMap, probes: crate::ProbeExecutor, + intents: intents::ServiceIntents, } impl NodeHealth { @@ -24,7 +26,9 @@ impl NodeHealth { store: StatusStore::new(&profile.paths.run_root)?, status: MonitorStatus::new(identity.uuid(), MonitorPhase::Initializing), attempts: BTreeMap::new(), + probe_failures: BTreeMap::new(), probes: crate::ProbeExecutor::new(true)?, + intents: intents::ServiceIntents::default(), }) } @@ -54,61 +58,7 @@ impl NodeHealth { } } self.reconcile_services(profile, processes).await?; - self.status.services.clear(); - let retained = super::services::retained(profile)?; - let mut ids = vec!["web".to_owned(), "kv".to_owned()]; - ids.extend(retained.iter().cloned().map(|intent| intent.service_id)); - for id in &ids { - if processes.owns(id) { - let mut healthy = processes.alive(id)?; - if healthy { - if id == "web" || id == "kv" { - let service = if id == "web" { - web.clone() - } else { - super::service(profile, "kv")? - }; - healthy = self - .probes - .probe_service(&service, &BTreeMap::new()) - .await - .is_ok(); - } else if let Some(intent) = retained - .iter() - .find(|intent| intent.service_id == *id && intent.kind != "kv") - { - healthy = match super::probes::for_intent(profile, intent) { - Ok(service) => self - .probes - .probe_service(&service, &BTreeMap::new()) - .await - .is_ok(), - Err(_) => false, - }; - } - } - self.status.services.insert( - id.clone(), - ServiceStatus { - operation_id: retained - .iter() - .find(|intent| intent.service_id == *id || (id == "kv" && intent.kind == "kv")) - .map(|intent| intent.operation_id.clone()), - pid: processes.pid(id), - generation: 1 + u64::from(*self.attempts.get(id.as_str()).unwrap_or(&0)), - healthy, - restart_attempts: *self.attempts.get(id.as_str()).unwrap_or(&0), - }, - ); - } - } - self.status.phase = if self.status.services.values().all(|service| service.healthy) { - MonitorPhase::Ready - } else { - MonitorPhase::Restarting - }; - self.store.publish(&mut self.status)?; - Ok(()) + self.observe(profile, web, processes).await } pub(in crate::node::runtime) async fn reconcile_services( @@ -116,27 +66,47 @@ impl NodeHealth { profile: &DeploymentProfile, processes: &mut ProcessManager, ) -> Result<()> { - if !profile.paths.data_root.join("node-binding.json").exists() { - return Ok(()); + if let Some(intents) = self.intents.poll(profile).await { + self.apply_services(profile, processes, &intents).await?; } - let intents = match super::services::desired(profile).await { - Ok(intents) => intents, - Err(error) => { - tracing::debug!(%error, "service recovery awaits committed intents"); - return Ok(()); - } - }; - for intent in &intents { - if !super::services::pending(profile, intent, processes)? { + Ok(()) + } + + async fn apply_services( + &mut self, + profile: &DeploymentProfile, + processes: &mut ProcessManager, + intents: &[crowdb_protocol::mgmt::node::NodeServiceIntent], + ) -> Result<()> { + for intent in intents { + let pending = super::services::pending(profile, intent, processes)?; + let restart_unhealthy = if !pending + && matches!( + intent.action, + crowdb_protocol::mgmt::node::NodeServiceAction::Start + | crowdb_protocol::mgmt::node::NodeServiceAction::Restart + ) { + self.restart_if_unhealthy(profile, intent, processes).await? + } else { + false + }; + if !pending && !restart_unhealthy { continue; } - if let Err(error) = super::services::dependencies_ready(profile, intent, &intents).await { + if let Err(error) = super::services::dependencies_ready(profile, intent, intents).await { tracing::debug!(service = %intent.service_id, %error, "service recovery waits for dependencies"); continue; } - match super::services::execute(profile, processes, intent.clone()).await { + if restart_unhealthy { + if let Err(error) = super::services::stop_unhealthy(profile, processes, intent).await { + tracing::debug!(service = %intent.service_id, %error, "unhealthy service recovery awaits current authority"); + continue; + } + } + match super::services::reconcile(profile, processes, intent.clone()).await { Ok(_) => { *self.attempts.entry(intent.service_id.clone()).or_default() += 1; + self.probe_failures.remove(&intent.service_id); } Err(error) => eprintln!("service recovery deferred for {}: {error}", intent.service_id), } @@ -144,6 +114,59 @@ impl NodeHealth { Ok(()) } + pub(super) async fn stop(&mut self) { + self.intents.stop().await; + } + + #[cfg(feature = "test-util")] + pub(in crate::node::runtime) async fn reconcile_services_for_tests( + &mut self, + profile: &DeploymentProfile, + processes: &mut ProcessManager, + ) -> Result<()> { + let intents = super::services::desired(profile).await?; + self.apply_services(profile, processes, &intents).await + } + + async fn restart_if_unhealthy( + &mut self, + profile: &DeploymentProfile, + intent: &crowdb_protocol::mgmt::node::NodeServiceIntent, + processes: &mut ProcessManager, + ) -> Result { + let process_id = if intent.kind == "kv" { + "kv" + } else { + &intent.service_id + }; + if !processes.owns(process_id) || !processes.alive(process_id)? { + self.probe_failures.remove(&intent.service_id); + return Ok(false); + } + // An ownership handover can withdraw readiness while the service is + // still advancing its durable transition. Restart only a failed live + // listener; keep readiness probes for the published health snapshot. + let service = match super::probes::liveness_for_intent(profile, intent) { + Ok(service) => service, + Err(error) => { + tracing::debug!(service = %intent.service_id, %error, "service health probe unavailable"); + return Ok(false); + } + }; + if self + .probes + .probe_service(&service, &BTreeMap::new()) + .await + .is_ok() + { + self.probe_failures.remove(&intent.service_id); + return Ok(false); + } + let failures = self.probe_failures.entry(intent.service_id.clone()).or_default(); + *failures = failures.saturating_add(1); + Ok(*failures >= service.probe.failure_threshold) + } + fn restart_allowed(&mut self, id: &str) -> Result<()> { let attempts = self.attempts.entry(id.into()).or_default(); if *attempts >= 5 { diff --git a/container/crowdb-monitor/src/node/runtime/health/intents.rs b/container/crowdb-monitor/src/node/runtime/health/intents.rs new file mode 100644 index 000000000..dab29c1d5 --- /dev/null +++ b/container/crowdb-monitor/src/node/runtime/health/intents.rs @@ -0,0 +1,96 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Read-only authority work cannot hold up current local health observations. + +use std::time::{Duration, Instant}; + +use crowdb_protocol::mgmt::node::{NodeBinding, NodeServiceIntent}; + +use super::super::{durable, services, Result}; +use crate::DeploymentProfile; + +struct Observation { + binding: NodeBinding, + started: Instant, + intents: Vec, +} + +#[derive(Default)] +pub(super) struct ServiceIntents { + task: Option>>, +} + +impl ServiceIntents { + pub(super) async fn poll(&mut self, profile: &DeploymentProfile) -> Option> { + let binding = match durable::read::(&profile.paths.data_root.join("node-binding.json")) { + Ok(Some(binding)) => binding, + Ok(None) => { + self.stop().await; + return None; + } + Err(error) => { + tracing::debug!(%error, "service intent observation awaits a valid local binding"); + return None; + } + }; + let mut ready = None; + if self + .task + .as_ref() + .is_some_and(tokio::task::JoinHandle::is_finished) + { + let task = self.task.take()?; + match task.await { + Ok(Ok(observation)) + if observation.binding == binding + && observation.started.elapsed() <= Duration::from_secs(3) => + { + ready = Some(observation.intents); + } + Ok(Ok(_)) => tracing::debug!("discarded expired or rebound service intent observation"), + Ok(Err(error)) => tracing::debug!(%error, "service recovery awaits committed intents"), + Err(error) => { + tracing::warn!(%error, "service intent worker failed; next health pass starts another read"); + } + } + } + if self.task.is_none() { + let profile = profile.clone(); + self.task = Some(tokio::spawn(async move { + let started = Instant::now(); + let intents = services::desired(&profile).await?; + if durable::read::(&profile.paths.data_root.join("node-binding.json"))?.as_ref() + != Some(&binding) + { + return Err("service intent binding changed during observation".into()); + } + Ok(Observation { + binding, + started, + intents, + }) + })); + } + ready + } + + pub(super) async fn stop(&mut self) { + if let Some(task) = self.task.take() { + task.abort(); + if let Err(error) = task.await { + if !error.is_cancelled() { + tracing::warn!(%error, "service intent worker stopped unexpectedly"); + } + } + } + } +} + +impl Drop for ServiceIntents { + fn drop(&mut self) { + if let Some(task) = &self.task { + task.abort(); + } + } +} diff --git a/container/crowdb-monitor/src/node/runtime/health/observations.rs b/container/crowdb-monitor/src/node/runtime/health/observations.rs new file mode 100644 index 000000000..f9b170c2f --- /dev/null +++ b/container/crowdb-monitor/src/node/runtime/health/observations.rs @@ -0,0 +1,76 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Read-only probes share a health pass instead of serializing their deadlines. + +use super::{NodeHealth, Result}; +use crate::{DeploymentProfile, MonitorPhase, ProcessManager, ServiceProfile, ServiceStatus}; +use std::collections::BTreeMap; + +impl NodeHealth { + pub(super) async fn observe( + &mut self, + profile: &DeploymentProfile, + web: &ServiceProfile, + processes: &mut ProcessManager, + ) -> Result<()> { + let retained = super::super::services::retained(profile)?; + let mut ids = vec!["web".to_owned(), "kv".to_owned()]; + ids.extend(retained.iter().map(|intent| intent.service_id.clone())); + let mut jobs = Vec::new(); + for id in ids { + if !processes.owns(&id) { + continue; + } + let intent = retained + .iter() + .find(|intent| intent.service_id == id || (id == "kv" && intent.kind == "kv")); + let service = match id.as_str() { + "web" => Some(web.clone()), + "kv" => Some(super::super::service(profile, "kv")?), + _ => intent.and_then(|intent| super::super::probes::for_intent(profile, intent).ok()), + }; + let attempts = *self.attempts.get(&id).unwrap_or(&0); + jobs.push(( + id.clone(), + service, + ServiceStatus { + operation_id: intent.map(|intent| intent.operation_id.clone()), + pid: processes.pid(&id), + generation: 1 + u64::from(attempts), + healthy: processes.alive(&id)?, + restart_attempts: attempts, + }, + )); + } + let probes = &self.probes; + let observations = + futures::future::join_all(jobs.into_iter().map(|(id, service, mut status)| async move { + status.healthy = if status.healthy { + match service { + Some(service) => probes.probe_service(&service, &BTreeMap::new()).await.is_ok(), + None => false, + } + } else { + false + }; + (id, status) + })) + .await; + self.status.services.clear(); + for (id, mut status) in observations { + let alive = processes.alive(&id)?; + let current_pid = processes.pid(&id); + status.healthy &= alive && current_pid == status.pid; + status.pid = current_pid; + self.status.services.insert(id, status); + } + self.status.phase = if self.status.services.values().all(|service| service.healthy) { + MonitorPhase::Ready + } else { + MonitorPhase::Restarting + }; + self.store.publish(&mut self.status)?; + Ok(()) + } +} diff --git a/container/crowdb-monitor/src/node/runtime/kv.rs b/container/crowdb-monitor/src/node/runtime/kv.rs index aa92efc66..8af9ed7de 100644 --- a/container/crowdb-monitor/src/node/runtime/kv.rs +++ b/container/crowdb-monitor/src/node/runtime/kv.rs @@ -55,6 +55,19 @@ pub(super) fn node_profile(config: &super::NodeRuntimeConfig) -> Result Result<(u16, u16)> { + let policy = crate::node::read_node_policy(root)?.ok_or("accepted node policy is absent")?; + policy.setup.paxos_kv_ports().ok_or_else(|| { + "accepted node policy has no enabled Paxos KV listeners" + .to_owned() + .into() + }) +} + +pub(super) fn ports(profile: &DeploymentProfile) -> Result<(u16, u16)> { + ports_from_root(&profile.paths.data_root) +} + pub(super) async fn bind( profile: &DeploymentProfile, processes: &mut ProcessManager, @@ -99,18 +112,19 @@ pub(super) async fn start_kv( accepted: &AcceptedNode, ) -> Result<()> { ensure_process(profile, processes, accepted).await?; - wait_ready(processes).await + wait_ready(profile, processes).await } -async fn wait_ready(processes: &mut ProcessManager) -> Result<()> { +async fn wait_ready(profile: &DeploymentProfile, processes: &mut ProcessManager) -> Result<()> { let client = reqwest::Client::builder() .no_proxy() .timeout(Duration::from_secs(1)) .build()?; + let (management_port, _) = ports(profile)?; let deadline = tokio::time::Instant::now() + Duration::from_secs(10); loop { if client - .get("http://127.0.0.1:10000/health") + .get(format!("http://127.0.0.1:{management_port}/health")) .send() .await .is_ok_and(|reply| reply.status().is_success()) @@ -143,7 +157,7 @@ pub(super) async fn start_for_cleanup( accepted: &AcceptedNode, ) -> Result<()> { spawn(profile, processes, accepted).await?; - wait_ready(processes).await + wait_ready(profile, processes).await } async fn spawn( @@ -168,15 +182,19 @@ async fn spawn( .join("kv") .join(format!("node-{}", accepted.node_id)); std::fs::create_dir_all(&root)?; + let (management_port, rpc_port) = ports(profile)?; + let rpc_health_port = rpc_port + .checked_add(1) + .ok_or("Paxos KV RPC listener has no health port")?; kv.args = vec![ "--node-recovery-root".into(), profile.paths.data_root.to_string_lossy().into_owned(), "--root".into(), root.to_string_lossy().into_owned(), "--management-port".into(), - "10000".into(), + management_port.to_string(), "--ports".into(), - "10100,10101".into(), + format!("{rpc_port},{rpc_health_port}"), "--instance-id".into(), accepted.node_id.to_string(), "--node-id".into(), diff --git a/container/crowdb-monitor/src/node/runtime/observation.rs b/container/crowdb-monitor/src/node/runtime/observation.rs index 04ad92d0b..d9d1e8488 100644 --- a/container/crowdb-monitor/src/node/runtime/observation.rs +++ b/container/crowdb-monitor/src/node/runtime/observation.rs @@ -35,6 +35,10 @@ pub(super) fn processes( // This authenticated read observes only local children. It needs no live // Group-0 grant and remains available while that authority is unavailable. let live = manager.live_pids()?; - let failures = super::services::failures(root, &live)?; - Ok(json!({"cluster_id":cluster,"node_id":node,"processes":live,"service_failures":failures})) + let failures = super::services::failures(root, &live, &binding)?; + let operations = super::services::process_operations(root, &live, &binding)?; + Ok( + json!({"cluster_id":cluster,"node_id":node,"bootstrap":binding.bootstrap, + "processes":live,"process_operations":operations,"service_failures":failures}), + ) } diff --git a/container/crowdb-monitor/src/node/runtime/probes.rs b/container/crowdb-monitor/src/node/runtime/probes.rs index d180f70ea..b5e681dfb 100644 --- a/container/crowdb-monitor/src/node/runtime/probes.rs +++ b/container/crowdb-monitor/src/node/runtime/probes.rs @@ -56,6 +56,29 @@ pub(super) fn for_intent(profile: &DeploymentProfile, intent: &NodeServiceIntent Ok(service) } +pub(super) fn liveness_for_intent( + profile: &DeploymentProfile, + intent: &NodeServiceIntent, +) -> Result { + let mut service = for_intent(profile, intent)?; + if matches!(service.probe.kind, ProbeKind::Http) { + let address = service + .probe + .target + .strip_suffix("/ready") + .ok_or("missing managed readiness path")?; + service.probe.target = format!( + "{address}{}", + if intent.kind == "access" { + "/live" + } else { + "/health" + } + ); + } + Ok(service) +} + fn field<'a>(mut value: &'a toml::Value, path: &[&str]) -> Result<&'a toml::Value> { for key in path { value = value.get(*key).ok_or("missing service probe setting")?; diff --git a/container/crowdb-monitor/src/node/runtime/recovery.rs b/container/crowdb-monitor/src/node/runtime/recovery.rs index 22ac60f34..6e6cc423c 100644 --- a/container/crowdb-monitor/src/node/runtime/recovery.rs +++ b/container/crowdb-monitor/src/node/runtime/recovery.rs @@ -13,14 +13,17 @@ use ssh_key::PublicKey; use super::{durable, AcceptedNode, Result}; use crate::NodeIdentity; +mod authority_refresh; mod confirmed; mod discovery; +#[cfg(feature = "test-util")] +pub use authority_refresh::recovery_authority_work_for_tests; mod owners; mod publication; pub(super) use confirmed::{accept as accept_confirmed, prepare_admitted}; pub(super) use discovery::start; #[cfg(feature = "test-util")] -pub use discovery::unique_member_endpoints_for_tests; +pub use discovery::{member_recovery_destinations_for_tests, unique_member_endpoints_for_tests}; pub(super) fn publish_local(root: &Path, addresses: &[std::net::IpAddr]) -> Result<()> { durable::write( @@ -266,7 +269,7 @@ pub fn member_trust_ready_for_tests(root: &Path) -> Result { } #[cfg(feature = "test-util")] -pub use publication::rebind_recovery_services_for_tests; +pub use publication::{apply_member_endpoints_for_tests, rebind_recovery_services_for_tests}; /// Read a live monitor's retained identity without waiting for lifecycle control. /// diff --git a/container/crowdb-monitor/src/node/runtime/recovery/authority_refresh.rs b/container/crowdb-monitor/src/node/runtime/recovery/authority_refresh.rs new file mode 100644 index 000000000..960d4e887 --- /dev/null +++ b/container/crowdb-monitor/src/node/runtime/recovery/authority_refresh.rs @@ -0,0 +1,35 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Shared deadline for recovered-route publication and optional contact refresh. + +use super::super::Result; +use std::future::Future; +use std::time::Duration; + +pub(super) async fn run( + publication: impl Future>, + confirmation: impl Future>, +) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(3); + // Recovered routes must reach current authority before optional reads + // through retained contacts can consume the shared deadline. + match tokio::time::timeout_at(deadline, publication).await { + Ok(Ok(())) => {} + Ok(Err(error)) => tracing::debug!(%error, "recovered endpoint publication awaits authority"), + Err(_) => tracing::debug!("recovered endpoint publication yielded at its deadline"), + } + match tokio::time::timeout_at(deadline, confirmation).await { + Ok(Ok(())) => {} + Ok(Err(error)) => tracing::debug!(%error, "confirmed recovery contacts await current authority"), + Err(_) => tracing::debug!("confirmed recovery contacts yielded at their deadline"), + } +} + +#[cfg(feature = "test-util")] +pub async fn recovery_authority_work_for_tests( + publication: impl Future>, + confirmation: impl Future>, +) { + run(publication, confirmation).await; +} diff --git a/container/crowdb-monitor/src/node/runtime/recovery/discovery.rs b/container/crowdb-monitor/src/node/runtime/recovery/discovery.rs index 4311b1498..958f2b99e 100644 --- a/container/crowdb-monitor/src/node/runtime/recovery/discovery.rs +++ b/container/crowdb-monitor/src/node/runtime/recovery/discovery.rs @@ -25,20 +25,20 @@ impl Drop for Worker { } } -pub(in crate::node::runtime) fn start(root: std::path::PathBuf, port: u16) -> Worker { +pub(in crate::node::runtime) fn start(root: std::path::PathBuf, port: u16, web_port: u16) -> Worker { Worker(tokio::spawn(async move { let mut tick = tokio::time::interval(Duration::from_secs(3)); tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); loop { tick.tick().await; - if let Err(error) = refresh(&root, port).await { + if let Err(error) = refresh(&root, port, web_port).await { tracing::debug!(%error, "member endpoint recovery is pending"); } } })) } -async fn refresh(root: &Path, port: u16) -> Result<()> { +async fn refresh(root: &Path, port: u16, web_port: u16) -> Result<()> { let Some(accepted) = durable::read::(&root.join("accepted-node.json"))? else { return Ok(()); }; @@ -52,9 +52,6 @@ async fn refresh(root: &Path, port: u16) -> Result<()> { { return Err("member recovery is retired".into()); } - if let Err(error) = super::confirmed::refresh(root, &accepted).await { - tracing::debug!(%error, "confirmed recovery contacts await current authority"); - } super::check(root, &accepted)?; let operation = PreparedBootstrap::load(&root.join("prepared-bootstrap.json"))?; let (trust, config) = super::confirmed::contacts(root, &operation)?; @@ -84,7 +81,7 @@ async fn refresh(root: &Path, port: u16) -> Result<()> { let (members, conflicts) = unique(endpoints); let snapshot = MemberEndpoints { version: 1, - bootstrap: accepted.bootstrap, + bootstrap: accepted.bootstrap.clone(), observer_boot_id: local.boot_id, observed_at_unix_ms: SystemTime::now() .duration_since(UNIX_EPOCH)? @@ -94,16 +91,11 @@ async fn refresh(root: &Path, port: u16) -> Result<()> { conflicts, }; durable::write(&root.join("member-endpoints.json"), &snapshot)?; - match tokio::time::timeout( - Duration::from_secs(3), - super::publication::reconcile(root, &operation, &snapshot, &config), - ) - .await - { - Ok(Ok(())) => {} - Ok(Err(error)) => tracing::debug!(%error, "recovered endpoint publication awaits authority"), - Err(_) => tracing::debug!("recovered endpoint publication yielded at its deadline"), - } + Box::pin(super::authority_refresh::run( + super::publication::reconcile(root, &operation, &snapshot, &config, web_port), + super::confirmed::refresh(root, &accepted), + )) + .await; Ok(()) } @@ -139,25 +131,52 @@ fn destinations( if destinations.len() >= 64 || !destinations.insert((member.node_id, host)) { continue; } - let mut node = config - .node(member.node_id) - .cloned() - .ok_or("member topology is absent")?; - node.host = host.to_string(); - node.ssh_password = None; - node.ssh_key = Some(root.join("ssh/id_ed25519").to_string_lossy().into_owned()); - probes.push(node); + probes.push(probe_node(root, config, member.node_id, host)?); + } + } + // Fresh discovery gets the bounded probe budget first so retained old + // addresses cannot crowd out a changed IP. Retained addresses are hints too: pinned SSH still proves the member + // before a route is published. Recovery must not depend on multicast or + // on Group 0 already being available after its majority restarts. + for member in trust.members.iter().take(64) { + if destinations.len() >= 64 { + break; + } + let node = config.node(member.node_id).ok_or("member topology is absent")?; + if let Ok(host) = node.host.parse::() { + if destinations.insert((member.node_id, host)) { + probes.push(probe_node(root, config, member.node_id, host)?); + } } } Ok(probes) } +fn probe_node( + root: &Path, + config: &ConsoleConfig, + node_id: u64, + host: std::net::IpAddr, +) -> Result { + let mut node = config.node(node_id).cloned().ok_or("member topology is absent")?; + node.host = host.to_string(); + node.ssh_password = None; + node.ssh_key = Some(root.join("ssh/id_ed25519").to_string_lossy().into_owned()); + Ok(node) +} + fn unique(endpoints: impl Iterator) -> (Vec, Vec) { let mut members = BTreeMap::::new(); let mut conflicts = BTreeSet::new(); for endpoint in endpoints { if let Some(prior) = members.get(&endpoint.node_id) { - if prior.boot_id != endpoint.boot_id || prior.discovery_id != endpoint.discovery_id { + // A node keeps its discovery identity across process restarts, but + // publishes a fresh boot id. The local observation is emitted + // before peer observations, so retain that authenticated identity + // instead of treating the restart as a conflicting member and + // pinning the route to the old address. + if prior.discovery_id != endpoint.discovery_id { + conflicts.insert(prior.discovery_id.clone()); conflicts.insert(endpoint.discovery_id.clone()); } } else { @@ -175,3 +194,15 @@ pub fn unique_member_endpoints_for_tests( ) -> (Vec, Vec) { unique(endpoints.into_iter()) } + +#[cfg(feature = "test-util")] +/// # Errors +/// Rejects trusted members missing from the retained topology. +pub fn member_recovery_destinations_for_tests( + root: &Path, + config: &ConsoleConfig, + trust: &MemberTrust, + snapshot: CandidateSnapshot, +) -> Result> { + destinations(root, config, trust, snapshot) +} diff --git a/container/crowdb-monitor/src/node/runtime/recovery/publication.rs b/container/crowdb-monitor/src/node/runtime/recovery/publication.rs index dacd623ea..a72a043fc 100644 --- a/container/crowdb-monitor/src/node/runtime/recovery/publication.rs +++ b/container/crowdb-monitor/src/node/runtime/recovery/publication.rs @@ -18,31 +18,11 @@ pub(super) async fn reconcile( operation: &PreparedBootstrap, endpoints: &MemberEndpoints, topology: &ConsoleConfig, + web_port: u16, ) -> Result<()> { let local = super::identity(root)?.endpoint; let mut config = topology.clone(); - let mut seeds = Vec::new(); - for node in &mut config.nodes { - let Some(peer) = endpoints.members.iter().find(|peer| peer.node_id == node.id) else { - continue; - }; - let address = peer - .addresses - .iter() - .find(|address| address.is_ipv4()) - .or_else(|| peer.addresses.first()) - .ok_or("member address missing")?; - node.host = address.to_string(); - seeds.push(format!("http://{}", std::net::SocketAddr::new(*address, 10000))); - for server in config - .servers - .iter_mut() - .filter(|server| server.node_id == Some(node.id)) - { - server.url = format!("http://{}", std::net::SocketAddr::new(*address, 10000)); - server.rpc_url = Some(std::net::SocketAddr::new(*address, 10100).to_string()); - } - } + let seeds = apply_member_endpoints(&mut config, endpoints)?; let client = Arc::new(CrowdbKvClient::new(ClientConfig::new(seeds.clone()))); let GetOutcome::Found { value, .. } = client .get( @@ -81,8 +61,9 @@ pub(super) async fn reconcile( .ok_or("local node topology missing")? .host .clone(); - if record.host != current { + if record.host != current || record.web_port != web_port { record.host.clone_from(¤t); + record.web_port = web_port; node_update::apply(&ctx, &operation.identity.cluster_id, record).await?; } super::owners::capture(root, &ctx, &operation.identity, local.node_id).await?; @@ -117,6 +98,41 @@ pub(super) async fn reconcile( Ok(()) } +fn apply_member_endpoints(config: &mut ConsoleConfig, endpoints: &MemberEndpoints) -> Result> { + let mut seeds = Vec::new(); + for node in &mut config.nodes { + let Some(peer) = endpoints.members.iter().find(|peer| peer.node_id == node.id) else { + continue; + }; + let address = peer + .addresses + .iter() + .find(|address| address.is_ipv4()) + .or_else(|| peer.addresses.first()) + .ok_or("member address missing")?; + node.host = address.to_string(); + let server = config + .servers + .iter_mut() + .find(|server| { + server.node_id == Some(node.id) + && server.service_type == crowdb_console_shared::config::ServiceType::PaxosKv + }) + .ok_or("member KV topology missing")?; + let management_port = reqwest::Url::parse(&server.url)? + .port_or_known_default() + .ok_or("member management port missing")?; + let rpc = server.rpc_url.as_ref().ok_or("member RPC endpoint missing")?; + let rpc_port = reqwest::Url::parse(&format!("http://{rpc}"))? + .port() + .ok_or("member RPC port missing")?; + server.url = format!("http://{}", std::net::SocketAddr::new(*address, management_port)); + server.rpc_url = Some(std::net::SocketAddr::new(*address, rpc_port).to_string()); + seeds.push(server.url.clone()); + } + Ok(seeds) +} + async fn rebind( root: &Path, client: &CrowdbKvClient, @@ -199,6 +215,18 @@ async fn rebind( Ok(true) } +#[cfg(feature = "test-util")] +/// Rebind authenticated contacts while retaining each member's accepted ports. +/// +/// # Errors +/// Rejects missing member topology or malformed accepted endpoints. +pub fn apply_member_endpoints_for_tests( + config: &mut ConsoleConfig, + endpoints: &MemberEndpoints, +) -> Result> { + apply_member_endpoints(config, endpoints) +} + #[cfg(feature = "test-util")] /// Exercise contact publication with no reachable authority. /// diff --git a/container/crowdb-monitor/src/node/runtime/services.rs b/container/crowdb-monitor/src/node/runtime/services.rs index 2664eaae2..d242000b1 100644 --- a/container/crowdb-monitor/src/node/runtime/services.rs +++ b/container/crowdb-monitor/src/node/runtime/services.rs @@ -11,6 +11,8 @@ use std::time::Duration; #[derive(Serialize, Deserialize)] pub(super) struct Execution { pub(super) intent: NodeServiceIntent, + #[serde(default)] + bootstrap: Option, complete: bool, pid: Option, #[serde(default)] @@ -28,7 +30,10 @@ mod execution; pub(super) use dependencies::ready as dependencies_ready; mod observation; mod policy; +mod receipt; pub(super) use execution::execute; +pub(super) use execution::reconcile; +pub(super) use execution::stop_unhealthy; pub(super) use observation::observe; pub(super) fn retained(profile: &DeploymentProfile) -> Result> { @@ -129,6 +134,11 @@ pub(super) fn pending( &intent.service_id }; let alive = processes.owns(id) && processes.alive(id)?; + let binding: NodeBinding = + durable::read(&profile.paths.data_root.join("node-binding.json"))?.ok_or("node is unbound")?; + if previous.bootstrap.as_ref() != Some(&binding.bootstrap) { + return Ok(true); + } if previous.attempts >= 5 && !alive && matches!( @@ -144,7 +154,7 @@ pub(super) fn pending( Ok(matches!( intent.action, NodeServiceAction::Start | NodeServiceAction::Restart - ) && !alive) + ) && (!alive || previous.pid != processes.pid(id))) } fn bounded_error(error: &str) -> String { @@ -154,6 +164,7 @@ fn bounded_error(error: &str) -> String { pub(super) fn failures( root: &std::path::Path, live: &std::collections::BTreeMap, + binding: &NodeBinding, ) -> Result { let directory = root.join("services"); let mut failures = serde_json::Map::new(); @@ -166,6 +177,12 @@ pub(super) fn failures( continue; } if let Some(execution) = durable::read::(&entry.path().join("execution.json"))? { + if execution.bootstrap.as_ref() != Some(&binding.bootstrap) + || execution.intent.cluster_id != binding.bootstrap.cluster_id + || execution.intent.node_id != binding.node_id + { + continue; + } if let Some(error) = execution.error { let process = if execution.intent.kind == "kv" { "kv" @@ -188,3 +205,42 @@ pub(super) fn failures( } Ok(Value::Object(failures)) } + +/// Couple each live managed PID to the durable execution that actually owns it. +pub(super) fn process_operations( + root: &std::path::Path, + live: &std::collections::BTreeMap, + binding: &NodeBinding, +) -> Result { + let directory = root.join("services"); + let mut operations = serde_json::Map::new(); + if !directory.exists() { + return Ok(Value::Object(operations)); + } + for entry in std::fs::read_dir(directory)? { + let entry = entry?; + if !entry.file_type()?.is_dir() { + continue; + } + if let Some(execution) = durable::read::(&entry.path().join("execution.json"))? { + if execution.bootstrap.as_ref() != Some(&binding.bootstrap) + || execution.intent.cluster_id != binding.bootstrap.cluster_id + || execution.intent.node_id != binding.node_id + { + continue; + } + let process = if execution.intent.kind == "kv" { + "kv" + } else { + &execution.intent.service_id + }; + if execution.pid.is_some() && execution.pid == live.get(process).copied() { + if operations.len() >= 4096 { + return Err("service process observation exceeds bound".into()); + } + operations.insert(process.to_owned(), execution.intent.operation_id.into()); + } + } + } + Ok(Value::Object(operations)) +} diff --git a/container/crowdb-monitor/src/node/runtime/services/dependencies.rs b/container/crowdb-monitor/src/node/runtime/services/dependencies.rs index d1540208e..6db53985b 100644 --- a/container/crowdb-monitor/src/node/runtime/services/dependencies.rs +++ b/container/crowdb-monitor/src/node/runtime/services/dependencies.rs @@ -83,6 +83,7 @@ pub(in crate::node::runtime) async fn ready( .await??; } if intent.kind == "access" { + use crowdb_chunk_kv_client::ChunkKvRangeCatalogSource; // An idle local ChunkKV listener does not prove that the catalog owner // has finished WAL replay. Probe the same routed read Access needs. let binding: NodeBinding = super::durable::read(&profile.paths.data_root.join("node-binding.json"))? @@ -91,24 +92,36 @@ pub(in crate::node::runtime) async fn ready( crowdb_kv_client::ClientConfig::new(binding.management_seeds), )); let catalog = Arc::new(crowdb_chunk_kv_client::Group0ChunkKvRangeCatalogSource::from_shared(kv)); - let transport = Arc::new(crowdb_chunk_kv_client::ChunkKvRpcTransport::new(1, 1, 1)); + let (_, pages) = catalog.load().await?; + let mut owner_keys = BTreeMap::new(); + for entry in pages.iter().flat_map(|page| &page.entries) { + owner_keys + .entry(entry.owner.rpc_endpoint.clone()) + .or_insert_with(|| entry.range.start.clone()); + } + let max_owners = owner_keys.len().max(1); + let transport = Arc::new(crowdb_chunk_kv_client::ChunkKvRpcTransport::new(max_owners, 1, 1)); let client = crowdb_chunk_kv_client::ChunkKvClient::new( crowdb_chunk_kv_client::ClientConfig { operation_timeout: Duration::from_secs(1), // Keep normal routed-read redirects within this probe's deadline: // catalog ownership may change between discovery and the read. - max_owner_connections: 1, + max_owner_connections: max_owners, ..Default::default() }, catalog, transport, )?; - let response = client.get(b"/deployment/access-readiness".to_vec(), None).await?; - if !matches!( - response.result, - Ok(crowdb_protocol::chunk_kv::OperationResult::Value(_)) - ) { - return Err(format!("routed ChunkKV catalog is not ready: {:?}", response.result).into()); + // Access startup can read any metadata range. A point routed to one + // healthy owner cannot authorize a launch while another owner replays. + for key in owner_keys.into_values() { + let response = client.get(key, None).await?; + if !matches!( + response.result, + Ok(crowdb_protocol::chunk_kv::OperationResult::Value(_)) + ) { + return Err(format!("routed ChunkKV catalog is not ready: {:?}", response.result).into()); + } } } Ok(()) diff --git a/container/crowdb-monitor/src/node/runtime/services/execution.rs b/container/crowdb-monitor/src/node/runtime/services/execution.rs index 88f7d8a9c..283a46a52 100644 --- a/container/crowdb-monitor/src/node/runtime/services/execution.rs +++ b/container/crowdb-monitor/src/node/runtime/services/execution.rs @@ -7,10 +7,48 @@ use crowdb_protocol::mgmt::node::{NodeBinding, NodeServiceAction, NodeServiceInt use serde_json::{json, Value}; use std::time::Duration; +pub(in crate::node::runtime) async fn stop_unhealthy( + profile: &DeploymentProfile, + processes: &mut ProcessManager, + intent: &NodeServiceIntent, +) -> Result<()> { + let binding: NodeBinding = + durable::read(&profile.paths.data_root.join("node-binding.json"))?.ok_or("node is unbound")?; + if binding.node_id != intent.node_id || binding.bootstrap.cluster_id != intent.cluster_id { + return Err("service intent belongs to another node or cluster".into()); + } + tokio::time::timeout(Duration::from_secs(3), super::super::verify_binding(&binding)).await??; + verify_intent(&binding, intent).await?; + let id = if intent.kind == "kv" { + "kv" + } else { + &intent.service_id + }; + processes.stop(id, Duration::from_secs(10)).await?; + Ok(()) +} + pub(in crate::node::runtime) async fn execute( profile: &DeploymentProfile, processes: &mut ProcessManager, intent: NodeServiceIntent, +) -> Result { + execute_with_readiness(profile, processes, intent, true).await +} + +pub(in crate::node::runtime) async fn reconcile( + profile: &DeploymentProfile, + processes: &mut ProcessManager, + intent: NodeServiceIntent, +) -> Result { + execute_with_readiness(profile, processes, intent, false).await +} + +async fn execute_with_readiness( + profile: &DeploymentProfile, + processes: &mut ProcessManager, + intent: NodeServiceIntent, + wait_initial_ready: bool, ) -> Result { validate(&intent)?; super::policy::validate(&profile.paths.data_root, &intent)?; @@ -19,6 +57,12 @@ pub(in crate::node::runtime) async fn execute( if binding.node_id != intent.node_id || binding.bootstrap.cluster_id != intent.cluster_id { return Err("service intent belongs to another node or cluster".into()); } + let directory = profile.paths.data_root.join("services").join(&intent.service_id); + let state = directory.join("execution.json"); + let previous = durable::read::(&state)?; + if super::receipt::completed_kv_stop(profile, processes, &binding, &intent, previous.as_ref())? { + return Ok(json!({"pid": null, "operation_id": intent.operation_id})); + } tokio::time::timeout(Duration::from_secs(3), super::super::verify_binding(&binding)).await??; verify_intent(&binding, &intent).await?; let process_id = if intent.kind == "kv" { @@ -26,28 +70,14 @@ pub(in crate::node::runtime) async fn execute( } else { &intent.service_id }; - let directory = profile.paths.data_root.join("services").join(&intent.service_id); std::fs::create_dir_all(&directory)?; - let state = directory.join("execution.json"); - let mut attempts = 0; - let mut continuing = false; - if let Some(previous) = durable::read::(&state)? { - if previous.intent.operation_id == intent.operation_id { - if previous.intent != intent { - return Err("operation identity reused with other inputs".into()); - } - attempts = previous.attempts; - continuing = matches!( - intent.action, - NodeServiceAction::Start | NodeServiceAction::Restart - ) && processes.owns(process_id) - && processes.alive(process_id)?; - if previous.complete - && (previous.pid.is_none() || (processes.owns(process_id) && processes.alive(process_id)?)) - { - return Ok(json!({"pid": processes.pid(process_id), "operation_id": intent.operation_id})); - } - } + let Replay { + mut attempts, + continuing, + completed, + } = replay(previous.as_ref(), &binding, &intent, processes, process_id)?; + if completed { + return Ok(json!({"pid": processes.pid(process_id), "operation_id": intent.operation_id})); } if matches!( intent.action, @@ -68,6 +98,7 @@ pub(in crate::node::runtime) async fn execute( &state, &Execution { intent: intent.clone(), + bootstrap: Some(binding.bootstrap.clone()), complete: false, pid: None, attempts, @@ -79,7 +110,16 @@ pub(in crate::node::runtime) async fn execute( .await .map(|()| processes.pid(process_id)) } else { - apply(profile, processes, &binding, &intent, process_id, &directory).await + apply( + profile, + processes, + &binding, + &intent, + process_id, + &directory, + wait_initial_ready, + ) + .await }; let (complete, pid, error) = match &result { Ok(pid) => (true, *pid, None), @@ -93,6 +133,7 @@ pub(in crate::node::runtime) async fn execute( &state, &Execution { intent: intent.clone(), + bootstrap: Some(binding.bootstrap.clone()), complete, pid, attempts: if complete { 0 } else { attempts }, @@ -103,6 +144,50 @@ pub(in crate::node::runtime) async fn execute( Ok(json!({"pid": pid, "operation_id": intent.operation_id})) } +#[derive(Default)] +struct Replay { + attempts: u32, + continuing: bool, + completed: bool, +} + +fn replay( + previous: Option<&Execution>, + binding: &NodeBinding, + intent: &NodeServiceIntent, + processes: &mut ProcessManager, + process_id: &str, +) -> Result { + let Some(previous) = previous.filter(|previous| previous.intent.operation_id == intent.operation_id) + else { + return Ok(Replay::default()); + }; + if previous.intent != *intent { + return Err("operation identity reused with other inputs".into()); + } + if previous + .bootstrap + .as_ref() + .is_some_and(|bootstrap| bootstrap != &binding.bootstrap) + { + return Err("service execution belongs to another bootstrap".into()); + } + let continuing = matches!( + intent.action, + NodeServiceAction::Start | NodeServiceAction::Restart + ) && processes.owns(process_id) + && processes.alive(process_id)?; + let completed = previous.complete + && previous.bootstrap.as_ref() == Some(&binding.bootstrap) + && previous.pid == processes.pid(process_id) + && (previous.pid.is_none() || (processes.owns(process_id) && processes.alive(process_id)?)); + Ok(Replay { + attempts: previous.attempts, + continuing, + completed, + }) +} + async fn apply( profile: &DeploymentProfile, processes: &mut ProcessManager, @@ -110,6 +195,7 @@ async fn apply( intent: &NodeServiceIntent, process_id: &str, directory: &std::path::Path, + wait_initial_ready: bool, ) -> Result> { // Fence the admitted operation immediately before disrupting its service. // KV may provide the authority being read, so verify while it is still live. @@ -160,7 +246,13 @@ async fn apply( let pid = processes .start(&service, &std::collections::BTreeMap::new()) .await?; - wait_ready(profile, processes, intent).await?; + if wait_initial_ready { + wait_ready(profile, processes, intent).await?; + } else { + // Recovery observes once per pass. A live child retains its + // durable launch attempt while health and control make progress. + observe_ready(profile, intent).await?; + } Some(pid) } } else { @@ -172,7 +264,11 @@ async fn apply( // A live process has already had its initial startup window. Later health // observations must yield after one probe so control requests can make progress. async fn observe_ready(profile: &DeploymentProfile, intent: &NodeServiceIntent) -> Result<()> { - let service = super::super::probes::for_intent(profile, intent)?; + let service = if intent.kind == "kv" { + super::super::service(profile, "kv")? + } else { + super::super::probes::for_intent(profile, intent)? + }; crate::ProbeExecutor::new(intent.kind == "diskio")? .probe_service(&service, &std::collections::BTreeMap::new()) .await diff --git a/container/crowdb-monitor/src/node/runtime/services/receipt.rs b/container/crowdb-monitor/src/node/runtime/services/receipt.rs new file mode 100644 index 000000000..fcfa3ce14 --- /dev/null +++ b/container/crowdb-monitor/src/node/runtime/services/receipt.rs @@ -0,0 +1,46 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Completed stop receipts observe a prior effect; they never execute one. + +use super::{ + durable, DeploymentProfile, Execution, NodeServiceAction, NodeServiceIntent, ProcessManager, Result, +}; +use crowdb_protocol::mgmt::node::NodeBinding; + +pub(super) fn completed_kv_stop( + profile: &DeploymentProfile, + processes: &mut ProcessManager, + binding: &NodeBinding, + intent: &NodeServiceIntent, + previous: Option<&Execution>, +) -> Result { + let Some(previous) = previous else { + return Ok(false); + }; + if intent.kind != "kv" + || !matches!(intent.action, NodeServiceAction::Stop | NodeServiceAction::Delete) + || previous.intent != *intent + || previous.bootstrap.as_ref() != Some(&binding.bootstrap) + || !previous.complete + || previous.pid.is_some() + || processes.owns("kv") && processes.alive("kv")? + { + return Ok(false); + } + let root = &profile.paths.data_root; + let Some(accepted) = durable::read::(&root.join("accepted-node.json"))? + else { + return Ok(false); + }; + Ok(accepted.prepared + && accepted.node_id == binding.node_id + && accepted.bootstrap == binding.bootstrap + && !super::super::admission::is_cancelled(root, &accepted) + && !root + .join(format!( + "retired-bootstrap-{}.json", + binding.bootstrap.operation_id + )) + .exists()) +} diff --git a/container/crowdb-monitor/src/node/runtime/services/test_support.rs b/container/crowdb-monitor/src/node/runtime/services/test_support.rs index cf0d68968..00a83ad01 100644 --- a/container/crowdb-monitor/src/node/runtime/services/test_support.rs +++ b/container/crowdb-monitor/src/node/runtime/services/test_support.rs @@ -72,6 +72,16 @@ impl NodeServiceForTests { .await } + /// # Errors + /// Propagates owned test KV child startup failures. + pub async fn start_kv_child_for_tests(&mut self) -> Result { + let service = super::super::service(&self.profile, "kv")?; + Ok(self + .processes + .start(&service, &std::collections::BTreeMap::new()) + .await?) + } + /// # Errors /// Propagates failure to stop the owned KV child. pub async fn stop_kv_for_tests(&mut self) -> Result<()> { @@ -81,6 +91,19 @@ impl NodeServiceForTests { Ok(()) } + /// # Errors + /// Propagates failure to stop one fixture-owned managed child. + pub async fn stop_child_for_tests(&mut self, id: &str) -> Result<()> { + self.processes.stop(id, std::time::Duration::from_secs(1)).await?; + Ok(()) + } + + /// # Errors + /// Rejects superseded recovery observations before stopping an owned process. + pub async fn stop_unhealthy_for_tests(&mut self, intent: &NodeServiceIntent) -> Result<()> { + super::stop_unhealthy(&self.profile, &mut self.processes, intent).await + } + /// # Errors /// Propagates production authority, process and readiness errors. pub async fn execute_for_tests(&mut self, intent: NodeServiceIntent) -> Result { @@ -92,8 +115,13 @@ impl NodeServiceForTests { /// # Errors /// Propagates authority, journal and process observation errors. pub async fn reconcile_for_tests(&mut self) -> Result<()> { - super::super::health::NodeHealth::new(&self.profile)? - .reconcile_services(&self.profile, &mut self.processes) + if self.health.is_none() { + self.health = Some(super::super::health::NodeHealth::new(&self.profile)?); + } + self.health + .as_mut() + .ok_or("missing test health worker")? + .reconcile_services_for_tests(&self.profile, &mut self.processes) .await } @@ -106,6 +134,27 @@ impl NodeServiceForTests { /// # Errors /// Propagates bounded failure observation errors. pub fn failures_for_tests(&mut self) -> Result { - failures(&self.profile.paths.data_root, &self.processes.live_pids()?) + let binding = super::durable::read::( + &self.profile.paths.data_root.join("node-binding.json"), + )? + .ok_or("node is unbound")?; + failures( + &self.profile.paths.data_root, + &self.processes.live_pids()?, + &binding, + ) + } + + /// Read the production bound namespace of this fixture's actual children. + /// + /// # Errors + /// Propagates binding, process and durable observation failures. + pub fn owned_processes_for_tests(&mut self, cluster: &str, node: u64) -> Result { + super::super::observation::processes( + &self.profile.paths.data_root, + &mut self.processes, + cluster, + node, + ) } } diff --git a/container/crowdb-monitor/src/node/runtime/setup/execution.rs b/container/crowdb-monitor/src/node/runtime/setup/execution.rs index 720ed0ed4..d4c42e1d6 100644 --- a/container/crowdb-monitor/src/node/runtime/setup/execution.rs +++ b/container/crowdb-monitor/src/node/runtime/setup/execution.rs @@ -79,11 +79,7 @@ async fn advance(profile: &DeploymentProfile, state: &mut AcceptedSetup) -> Resu state.status.phase = NodeSetupPhase::Running; journal::save(root, state)?; let choices = &state.request.choices; - let preparation_needed = !matches!( - choices.storage, - crowdb_protocol::mgmt::node::setup::SetupStorage::Diskless - ) || choices.services.iter().any(|service| service.enabled); - if preparation_needed + if choices.requires_default_preparation() && crate::node::read_node_policy(root)?.is_some() && !metadata::ready(root, &binding, &kv).await? { diff --git a/container/crowdb-monitor/src/node/runtime/setup/execution/services.rs b/container/crowdb-monitor/src/node/runtime/setup/execution/services.rs index 9304d989c..cb267f82f 100644 --- a/container/crowdb-monitor/src/node/runtime/setup/execution/services.rs +++ b/container/crowdb-monitor/src/node/runtime/setup/execution/services.rs @@ -90,7 +90,7 @@ pub(super) async fn advance( } step.phase = SetupStepPhase::Preparing; journal::save(&profile.paths.data_root, state)?; - let (success, value) = prepare(binding, state, kind).await?; + let (success, value) = prepare(profile, binding, state, kind).await?; let step = &mut state.status.services[index]; step.phase = if success { SetupStepPhase::Accepted @@ -122,17 +122,24 @@ fn name(kind: SetupServiceKind) -> &'static str { } async fn prepare( + profile: &DeploymentProfile, binding: &NodeBinding, state: &AcceptedSetup, kind: SetupServiceKind, ) -> Result<(bool, serde_json::Value)> { + let web_port = profile + .public_endpoints + .iter() + .find(|endpoint| endpoint.id == "web") + .map(|endpoint| endpoint.port) + .ok_or("node profile requires a Web endpoint")?; let mut response = reqwest::Client::builder() .no_proxy() .timeout(Duration::from_secs(30)) .build()? .post(format!( - "http://127.0.0.1:9090/api/nodes/{}/setup/prepare-service", - binding.node_id + "http://127.0.0.1:{web_port}/api/nodes/{}/setup/prepare-service", + binding.node_id, )) .json(&json!({"operation_id":state.request.operation_id,"kind":kind})) .send() diff --git a/container/crowdb-monitor/src/node/startup.rs b/container/crowdb-monitor/src/node/startup.rs index 8c451a7ae..03377ff7d 100644 --- a/container/crowdb-monitor/src/node/startup.rs +++ b/container/crowdb-monitor/src/node/startup.rs @@ -10,7 +10,10 @@ use serde::{Deserialize, Serialize}; use super::durable; mod policy; -pub use policy::{accept_node_policy, read_node_policy}; +pub use policy::{ + accept_node_policy, accept_node_policy_with_port_offset, read_node_policy, + resolve_node_policy_with_port_offset, +}; #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] diff --git a/container/crowdb-monitor/src/node/startup/policy.rs b/container/crowdb-monitor/src/node/startup/policy.rs index 9ef1767b6..0148669f7 100644 --- a/container/crowdb-monitor/src/node/startup/policy.rs +++ b/container/crowdb-monitor/src/node/startup/policy.rs @@ -7,7 +7,7 @@ use std::{fs, io, path::Path}; use crowdb_protocol::mgmt::node::{ deployment::{NodeDeploymentOverrides, NodeDeploymentPolicy}, - setup::SetupStorage, + setup::{SetupService, SetupStorage}, }; use crate::node::durable; @@ -34,14 +34,30 @@ pub fn accept_node_policy( root: &Path, file: &NodeDeploymentOverrides, explicit: &NodeDeploymentOverrides, +) -> io::Result { + accept_node_policy_with_port_offset(root, file, explicit, 0) +} + +/// Accept a fresh policy after shifting every service listener by `offset`. +/// +/// The offset is a launcher concern: it is applied once, before the policy is +/// sealed, and retained unchanged on every restart. Existing accepted roots +/// are never shifted again. +/// +/// # Errors +/// Returns an error when the root, policy inputs or shifted listener ports are +/// invalid, or when a retained policy does not match the supplied inputs. +pub fn accept_node_policy_with_port_offset( + root: &Path, + file: &NodeDeploymentOverrides, + explicit: &NodeDeploymentOverrides, + offset: u16, ) -> io::Result { if !crate::layout::is_clean_absolute(root) || !fs::symlink_metadata(root)?.is_dir() { return Err(io::Error::other("node policy root must be an absolute directory")); } let accepted = read_node_policy(root)?; - let selected = explicit - .resolve(file, accepted.as_ref()) - .map_err(io::Error::other)?; + let selected = resolve_node_policy_with_port_offset(file, explicit, accepted.as_ref(), offset)?; if accepted.is_some() { validate_retained_identity(root)?; return Ok(selected); @@ -65,6 +81,51 @@ pub fn accept_node_policy( Ok(winner) } +/// Resolve launcher inputs into the same port space as the accepted policy. +/// +/// # Errors +/// Rejects invalid inputs, port overflow and changes to accepted policy. +pub fn resolve_node_policy_with_port_offset( + file: &NodeDeploymentOverrides, + explicit: &NodeDeploymentOverrides, + accepted: Option<&NodeDeploymentPolicy>, + offset: u16, +) -> io::Result { + let (mut file, mut explicit) = (file.clone(), explicit.clone()); + if accepted.is_some() { + for services in [&mut file.services, &mut explicit.services].into_iter().flatten() { + shift_service_ports(services, offset)?; + } + } + let mut selected = explicit.resolve(&file, accepted).map_err(io::Error::other)?; + if accepted.is_none() { + shift_service_ports(&mut selected.setup.services, offset)?; + selected.validate().map_err(io::Error::other)?; + } + Ok(selected) +} + +fn shift_service_ports(services: &mut [SetupService], offset: u16) -> io::Result<()> { + for service in services { + for port in [ + &mut service.listeners.listen_port, + &mut service.listeners.http_port, + &mut service.listeners.rpc_port, + &mut service.listeners.s3_port, + &mut service.listeners.dataset_port, + &mut service.listeners.health_port, + ] + .into_iter() + .flatten() + { + *port = port + .checked_add(offset) + .ok_or_else(|| io::Error::other("port offset exceeds 65535"))?; + } + } + Ok(()) +} + fn validate_retained_identity(root: &Path) -> io::Result<()> { if root.join("node-identity").try_exists()? { return Ok(()); diff --git a/container/crowdb-monitor/src/node_args.rs b/container/crowdb-monitor/src/node_args.rs index ef401d7e9..8f793b987 100644 --- a/container/crowdb-monitor/src/node_args.rs +++ b/container/crowdb-monitor/src/node_args.rs @@ -45,6 +45,8 @@ pub(crate) struct NodeRunArgs { pub(crate) async fn run(args: NodeRunArgs) -> Result<(), Box> { let inputs = resolve(&args)?; + let offset = port_offset()?; + let monitor_port = shifted_port(9095, offset)?; let addresses = if_addrs::get_if_addrs()? .into_iter() .filter(|address| args.interface.contains(&address.name)) @@ -52,11 +54,11 @@ pub(crate) async fn run(args: NodeRunArgs) -> Result<(), Box Result<(), Box Result> { + let value = std::env::var("CROWDB_PORT_OFFSET").unwrap_or_default(); + if value.is_empty() { + return Ok(0); + } + value + .parse::() + .map_err(|_| "CROWDB_PORT_OFFSET must be a non-negative integer fitting a port".into()) +} + +fn shifted_port(base: u16, offset: u16) -> Result> { + base.checked_add(offset) + .ok_or_else(|| "CROWDB_PORT_OFFSET shifts a listener beyond port 65535".into()) +} + pub(crate) fn accept(args: &NodeRunArgs) -> Result<&'static str, Box> { let inputs = resolve(args)?; let policy = crowdb_monitor::accept_node_policy(&inputs.data_root, &inputs.file, &inputs.explicit)?; @@ -102,12 +119,25 @@ fn resolve(args: &NodeRunArgs) -> Result>, } @@ -108,8 +109,14 @@ impl ProcessManager { let policy = self.log_policy.clone(); let logger = tokio::spawn(async move { Box::pin(log::pump(stdout, stderr, &log_directory, policy)).await }); - self.processes - .insert(service.id.clone(), ManagedProcess { child, logger }); + self.processes.insert( + service.id.clone(), + ManagedProcess { + spawn_pid: pid, + child, + logger, + }, + ); if let Err(error) = self .events .record(&MonitorEvent { @@ -198,24 +205,7 @@ impl ProcessManager { .processes .remove(id) .ok_or(ProcessError::Invalid("service is not owned"))?; - if process.child.try_wait()?.is_none() { - if let Some(pid) = process - .child - .id() - .and_then(|value| i32::try_from(value).ok()) - .and_then(Pid::from_raw) - { - if let Err(error) = kill_process(pid, Signal::TERM) { - if error != rustix::io::Errno::SRCH { - return Err(io::Error::from_raw_os_error(error.raw_os_error()).into()); - } - } - } - if timeout(grace, process.child.wait()).await.is_err() { - process.child.start_kill()?; - process.child.wait().await?; - } - } + let observation = exit_observation::stop(&mut process.child, process.spawn_pid, grace).await?; let log_result = timeout(Duration::from_secs(5), &mut process.logger).await; if let Ok(joined) = log_result { joined?.map_err(ProcessError::Io)?; @@ -223,6 +213,16 @@ impl ProcessManager { process.logger.abort(); return Err(ProcessError::Invalid("child log pipes did not close after exit")); } + if let Err(error) = self + .events + .record_process_exit(id, process.spawn_pid, &observation) + .await + { + eprintln!( + "process exit evidence was not retained: service={id} pid={} error={error}", + process.spawn_pid + ); + } self.events .record(&MonitorEvent { kind: MonitorEventKind::ChildStopped, diff --git a/container/crowdb-monitor/src/process/exit_observation.rs b/container/crowdb-monitor/src/process/exit_observation.rs new file mode 100644 index 000000000..c6ca809dc --- /dev/null +++ b/container/crowdb-monitor/src/process/exit_observation.rs @@ -0,0 +1,82 @@ +use std::io; +use std::os::unix::process::ExitStatusExt; +use std::process::ExitStatus; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use rustix::process::{kill_process, Pid, Signal}; +use tokio::process::Child; +use tokio::time::timeout; + +use super::ProcessError; + +pub(super) async fn stop( + child: &mut Child, + original_pid: u32, + grace: Duration, +) -> Result { + let mut status = child.try_wait()?; + let mut stop_path = "already_exited_before_stop"; + let mut term_sent_at_unix_ms = None; + let mut term_clock = None; + let mut kill_requested_at_unix_ms = None; + let mut wait_error = None; + if status.is_none() { + stop_path = "owned_stop_requested"; + if let Some(pid) = child + .id() + .and_then(|value| i32::try_from(value).ok()) + .and_then(Pid::from_raw) + { + let attempted_clock = Instant::now(); + let attempted_at = unix_ms(); + match kill_process(pid, Signal::TERM) { + Ok(()) => { + term_sent_at_unix_ms = attempted_at; + term_clock = Some(attempted_clock); + } + Err(error) if error == rustix::io::Errno::SRCH => {} + Err(error) => return Err(io::Error::from_raw_os_error(error.raw_os_error()).into()), + } + } + // No diagnostic I/O runs between TERM and this original grace wait. + match timeout(grace, child.wait()).await { + Ok(Ok(observed)) => status = Some(observed), + // The original stop path ignored this inner wait error. Retain + // that behavior, but do not manufacture a successful exit status. + Ok(Err(error)) => wait_error = Some(error.to_string()), + Err(_) => { + stop_path = "owned_stop_grace_elapsed"; + child.start_kill()?; + kill_requested_at_unix_ms = unix_ms(); + status = Some(child.wait().await?); + } + } + } + let wait_returned_at_unix_ms = unix_ms(); + let elapsed_since_term_ms = term_clock.map(|clock| clock.elapsed().as_millis()); + let observation = serde_json::json!({ + "original_pid": original_pid, + "stop_path": stop_path, + "status_observed": status.is_some(), + "exit_code": status.as_ref().and_then(ExitStatus::code), + "exit_signal": status.as_ref().and_then(ExitStatusExt::signal), + "core_dumped": status.as_ref().map(ExitStatusExt::core_dumped), + "success": status.as_ref().map(ExitStatus::success), + "raw_wait_status": status.as_ref().map(|value| (*value).into_raw()), + "term_sent_at_unix_ms": term_sent_at_unix_ms, + "kill_requested_at_unix_ms": kill_requested_at_unix_ms, + "wait_returned_at_unix_ms": wait_returned_at_unix_ms, + "elapsed_since_term_ms": elapsed_since_term_ms, + "grace_ms": grace.as_millis(), + "wait_error": wait_error, + "caller_stop_reason": "not supplied by existing ProcessManager::stop API", + }); + Ok(observation) +} + +fn unix_ms() -> Option { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .ok() + .map(|value| value.as_millis()) +} diff --git a/container/crowdb-monitor/tests/common/admitted_member.rs b/container/crowdb-monitor/tests/common/admitted_member.rs index 1c6cad06a..2ce7abc4a 100644 --- a/container/crowdb-monitor/tests/common/admitted_member.rs +++ b/container/crowdb-monitor/tests/common/admitted_member.rs @@ -22,7 +22,7 @@ pub fn fixture(root: &Path) -> ConfirmedMemberTrust { let nodes: Vec<_> = (1_u64..=3).map(|id| { config.nodes.push(serde_json::from_value(json!({"id":id,"rack_id":1,"host":format!("192.0.2.{id}"),"ssh_user":"crowdb"})).unwrap()); config.servers.push(serde_json::from_value(json!({"id":format!("kv-{id}"),"node_id":id,"url":format!("http://192.0.2.{id}:10000"),"rpc_url":format!("192.0.2.{id}:10100")})).unwrap()); - NodeRecord { discovery_id: uuid::Uuid::new_v4().to_string(), node_id:id, physical_host_id:format!("host-{id}"), + NodeRecord { web_port: 9090, discovery_id: uuid::Uuid::new_v4().to_string(), node_id:id, physical_host_id:format!("host-{id}"), rack_id:1, host:format!("192.0.2.{id}"), ssh_port:22, ssh_user:"crowdb".into(), operation_id:uuid::Uuid::new_v4().to_string(), confirmed:false, cancelled:false, host_key:Some(super::trust::key(u8::try_from(id).unwrap())) } @@ -62,6 +62,7 @@ pub fn fixture(root: &Path) -> ConfirmedMemberTrust { node.confirmed = true; } confirmed.push(NodeRecord { + web_port: 9090, discovery_id: own.discovery_id, node_id: 4, physical_host_id: "host-4".into(), diff --git a/container/crowdb-monitor/tests/common/managed_health.rs b/container/crowdb-monitor/tests/common/managed_health.rs new file mode 100644 index 000000000..6773f0e3c --- /dev/null +++ b/container/crowdb-monitor/tests/common/managed_health.rs @@ -0,0 +1,180 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crate::node_setup; +use crowdb_monitor::{DeploymentProfile, NodeIdentity, NodeServiceForTests}; +use crowdb_protocol::mgmt::node::{ + setup::{NodeSetupChoices, NodeSetupRequest}, + NodeServiceAction, NodeServiceIntent, +}; +use std::{ + collections::BTreeMap, + os::unix::fs::PermissionsExt, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, +}; + +pub struct TestManagedHealth { + pub worker: NodeServiceForTests, + pub ready: Arc, + pub slow: Arc, + pub intent: NodeServiceIntent, + pub profile: DeploymentProfile, + _authority: node_setup::TestAuthority, + server: tokio::task::JoinHandle<()>, + _directory: crowdb_test_harness::test_dirs::TestDir, +} + +impl TestManagedHealth { + pub async fn new() -> Self { + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("managed-health"); + let data = directory.path().join("data"); + std::fs::create_dir(&data).unwrap(); + let root = data.as_path(); + let request = NodeSetupRequest { + version: 1, + discovery_id: NodeIdentity::load_or_create(root).unwrap().uuid().to_string(), + operation_id: uuid::Uuid::new_v4().to_string(), + choices: NodeSetupChoices::default(), + }; + let authority = node_setup::TestAuthority::start(root, &request).await; + assert_eq!(authority.registry.nodes[0].node_id, 2); + let ready = Arc::new(AtomicBool::new(true)); + let slow = Arc::new(AtomicBool::new(false)); + let (address, server) = health_server(Arc::clone(&ready), Arc::clone(&slow)).await; + let mut profile = DeploymentProfile::load( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../oci-image/node-profile.toml"), + ) + .unwrap(); + profile.paths.data_root = root.to_path_buf(); + profile.paths.install_root = root.parent().unwrap().to_path_buf(); + profile.paths.bin_root = root.join("bin"); + profile.paths.template_root = root.join("templates"); + profile.paths.log_root = root.join("log"); + profile.paths.run_root = directory.path().join("run"); + std::fs::create_dir_all(&profile.paths.log_root).unwrap(); + std::fs::create_dir_all(&profile.paths.run_root).unwrap(); + std::fs::create_dir_all(&profile.paths.bin_root).unwrap(); + let program = profile.paths.bin_root.join("owned-child"); + std::fs::write(&program, "#!/bin/sh\nexec sleep 60\n").unwrap(); + std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o700)).unwrap(); + for service in &mut profile.services { + service.program.clone_from(&program); + service.config_template = None; + if matches!(service.id.as_str(), "web" | "kv" | "chunkdb") { + service.probe.target = format!("http://{address}/health"); + service.additional_probes.clear(); + // Keep the service executor and real Group-0 authority; this + // fixture isolates readiness from storage dependency assembly. + service.dependencies.clear(); + } + } + profile.validate().unwrap(); + let intent = NodeServiceIntent { + cluster_id: authority.binding.bootstrap.cluster_id.clone(), + operation_id: uuid::Uuid::new_v4().to_string(), + node_id: 2, + service_id: "chunkdb-2".into(), + kind: "chunkdb".into(), + action: NodeServiceAction::Start, + configuration: format!("[server]\nhttp_listen_addr = \"{address}\"\n"), + environment: BTreeMap::new(), + }; + authority + .kv + .put( + 0, + 0, + b"/deployment/services/2/chunkdb-2", + &serde_json::to_vec(&intent).unwrap(), + None, + ) + .await + .unwrap(); + let mut worker = NodeServiceForTests::new_for_tests(profile.clone()).await.unwrap(); + worker.start_kv_child_for_tests().await.unwrap(); + worker.execute_for_tests(intent.clone()).await.unwrap(); + Self { + worker, + ready, + slow, + intent, + profile, + _authority: authority, + server, + _directory: directory, + } + } + + pub fn execution(&self) -> serde_json::Value { + serde_json::from_slice( + &std::fs::read( + self.profile + .paths + .data_root + .join("services/chunkdb-2/execution.json"), + ) + .unwrap(), + ) + .unwrap() + } + + pub async fn stop(&mut self) { + for id in ["chunkdb-2", "kv", "web"] { + self.worker.stop_child_for_tests(id).await.unwrap(); + } + } +} + +impl Drop for TestManagedHealth { + fn drop(&mut self) { + self.server.abort(); + } +} + +async fn health_server( + ready: Arc, + slow: Arc, +) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let observed = Arc::clone(&ready); + let health_slow = Arc::clone(&slow); + let ready_slow = Arc::clone(&slow); + let server = tokio::spawn(async move { + let app = axum::Router::new() + .route( + "/health", + axum::routing::get(move || { + let slow = Arc::clone(&health_slow); + async move { + if slow.load(Ordering::Acquire) { + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + } + "healthy" + } + }), + ) + .route( + "/ready", + axum::routing::get(move || { + let ready = Arc::clone(&observed); + let slow = Arc::clone(&ready_slow); + async move { + if slow.load(Ordering::Acquire) { + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + } + if ready.load(Ordering::Acquire) { + axum::http::StatusCode::OK + } else { + axum::http::StatusCode::SERVICE_UNAVAILABLE + } + } + }), + ); + axum::serve(listener, app).await.unwrap(); + }); + (address, server) +} diff --git a/container/crowdb-monitor/tests/common/member_trust.rs b/container/crowdb-monitor/tests/common/member_trust.rs index 304be3256..6ffff6c9d 100644 --- a/container/crowdb-monitor/tests/common/member_trust.rs +++ b/container/crowdb-monitor/tests/common/member_trust.rs @@ -44,6 +44,7 @@ pub fn fixture(root: &Path) -> MemberTrust { ); config.servers.push(serde_json::from_value(json!({"id":format!("kv-{id}"),"node_id":id,"url":format!("http://192.0.2.{id}:10000"),"rpc_url":format!("192.0.2.{id}:10100")})).unwrap()); nodes.push(NodeRecord { + web_port: 9090, discovery_id: ids[usize::try_from(id - 1).unwrap()].clone(), node_id: id, physical_host_id: "test-host".into(), diff --git a/container/crowdb-monitor/tests/common/node_health.rs b/container/crowdb-monitor/tests/common/node_health.rs new file mode 100644 index 000000000..fb910fb87 --- /dev/null +++ b/container/crowdb-monitor/tests/common/node_health.rs @@ -0,0 +1,133 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use axum::{extract::State, http::StatusCode, routing::get, Router}; +use crowdb_monitor::{DeploymentProfile, MonitorStatus, NodeServiceForTests, StatusStore}; +use crowdb_protocol::mgmt::{node::NodeBinding, SystemBootstrapIdentity}; +use std::{ + os::unix::fs::PermissionsExt, + path::Path, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::Duration, +}; + +pub struct TestHealth { + pub worker: NodeServiceForTests, + pub ready: Arc, + pub entered: Arc, + profile: DeploymentProfile, + servers: Vec>, + _directory: crowdb_test_harness::test_dirs::TestDir, +} + +impl TestHealth { + pub async fn new() -> Self { + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-health-authority"); + let root = directory.path(); + let ready = Arc::new(AtomicBool::new(true)); + let entered = Arc::new(tokio::sync::Notify::new()); + let (health, health_task) = health_server(Arc::clone(&ready)).await; + let (authority, authority_task) = slow_authority(Arc::clone(&entered)).await; + let profile = profile(root, &health); + let binding = NodeBinding { + node_id: 2, + bootstrap: SystemBootstrapIdentity { + cluster_id: uuid::Uuid::new_v4().to_string(), + operation_id: uuid::Uuid::new_v4().to_string(), + configuration_digest: "00".repeat(32), + }, + management_seeds: vec![authority], + }; + let binding_path = root.join("node-binding.json"); + std::fs::write(&binding_path, serde_json::to_vec(&binding).unwrap()).unwrap(); + std::fs::set_permissions(binding_path, std::fs::Permissions::from_mode(0o600)).unwrap(); + let mut worker = NodeServiceForTests::new_for_tests(profile.clone()).await.unwrap(); + worker.start_kv_child_for_tests().await.unwrap(); + Self { + worker, + ready, + entered, + profile, + servers: vec![health_task, authority_task], + _directory: directory, + } + } + + pub fn snapshot(&self) -> MonitorStatus { + StatusStore::open(&self.profile.paths.run_root) + .unwrap() + .read(Duration::from_secs(3)) + .unwrap() + } +} + +impl Drop for TestHealth { + fn drop(&mut self) { + for server in &self.servers { + server.abort(); + } + } +} + +fn profile(root: &Path, health: &str) -> DeploymentProfile { + let mut profile = + DeploymentProfile::load(Path::new(env!("CARGO_MANIFEST_DIR")).join("../oci-image/node-profile.toml")) + .unwrap(); + profile.paths.data_root = root.to_path_buf(); + profile.paths.log_root = root.join("log"); + profile.paths.run_root = root.join("run"); + std::fs::create_dir_all(&profile.paths.log_root).unwrap(); + std::fs::create_dir_all(&profile.paths.run_root).unwrap(); + for service in &mut profile.services { + if matches!(service.id.as_str(), "web" | "kv") { + service.program = "/bin/sleep".into(); + service.args = vec!["60".into()]; + service.probe.target = format!("{health}/health"); + service.additional_probes.clear(); + } + } + profile +} + +async fn health_server(ready: Arc) -> (String, tokio::task::JoinHandle<()>) { + async fn health(State(ready): State>) -> StatusCode { + if ready.load(Ordering::Acquire) { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + } + } + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!("http://{}", listener.local_addr().unwrap()); + let task = tokio::spawn(async move { + axum::serve( + listener, + Router::new().route("/health", get(health)).with_state(ready), + ) + .await + .unwrap(); + }); + (origin, task) +} + +async fn slow_authority(entered: Arc) -> (String, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!("http://{}", listener.local_addr().unwrap()); + let task = tokio::spawn(async move { + let app = Router::new().route( + "/topology", + get(move || { + let entered = Arc::clone(&entered); + async move { + entered.notify_one(); + std::future::pending::().await + } + }), + ); + axum::serve(listener, app).await.unwrap(); + }); + (origin, task) +} diff --git a/container/crowdb-monitor/tests/common/node_setup.rs b/container/crowdb-monitor/tests/common/node_setup.rs index d8d7de409..31f8a27e4 100644 --- a/container/crowdb-monitor/tests/common/node_setup.rs +++ b/container/crowdb-monitor/tests/common/node_setup.rs @@ -54,6 +54,7 @@ impl TestAuthority { let registry = NodeRegistry { cluster_id: binding.bootstrap.cluster_id.clone(), nodes: vec![NodeRecord { + web_port: 9090, discovery_id: request.discovery_id.clone(), operation_id: request.operation_id.clone(), node_id: 2, diff --git a/container/crowdb-monitor/tests/common/node_setup_control.rs b/container/crowdb-monitor/tests/common/node_setup_control.rs index 38aa1acc7..9d9ac3b2b 100644 --- a/container/crowdb-monitor/tests/common/node_setup_control.rs +++ b/container/crowdb-monitor/tests/common/node_setup_control.rs @@ -6,6 +6,14 @@ use std::path::Path; use tokio::io::{AsyncReadExt, AsyncWriteExt}; pub fn processes_once(root: &Path) -> tokio::task::JoinHandle<()> { + processes_reply_once(root, false) +} + +pub fn processes_with_kv_once(root: &Path) -> tokio::task::JoinHandle<()> { + processes_reply_once(root, true) +} + +fn processes_reply_once(root: &Path, kv_running: bool) -> tokio::task::JoinHandle<()> { std::fs::create_dir_all(root).unwrap(); let listener = tokio::net::UnixListener::bind(root.join("node-control.sock")).unwrap(); tokio::spawn(async move { @@ -16,8 +24,15 @@ pub fn processes_once(root: &Path) -> tokio::task::JoinHandle<()> { serde_json::from_slice::(&bytes).unwrap(), NodeControl::Processes { .. } )); + let processes = if kv_running { + serde_json::json!({"kv":42}) + } else { + serde_json::json!({}) + }; + let reply = serde_json::json!({"ok":true,"value":{ + "processes":processes,"service_failures":{}}}); stream - .write_all(br#"{"ok":true,"value":{"processes":{},"service_failures":{}}}"#) + .write_all(&serde_json::to_vec(&reply).unwrap()) .await .unwrap(); }) diff --git a/container/crowdb-monitor/tests/common/service_receipt.rs b/container/crowdb-monitor/tests/common/service_receipt.rs new file mode 100644 index 000000000..fb32ac200 --- /dev/null +++ b/container/crowdb-monitor/tests/common/service_receipt.rs @@ -0,0 +1,107 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crate::node_setup; +use crowdb_monitor::{DeploymentProfile, NodeIdentity, NodeServiceForTests}; +use crowdb_protocol::mgmt::node::{ + setup::{NodeSetupChoices, NodeSetupRequest}, + NodeServiceAction, NodeServiceIntent, +}; +use crowdb_test_harness::test_dirs::TestDir; +use std::{ + collections::BTreeMap, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, +}; + +pub struct TestStoppedKv { + pub directory: TestDir, + pub worker: NodeServiceForTests, + pub intent: NodeServiceIntent, +} + +impl TestStoppedKv { + pub async fn new() -> Self { + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("completed-kv-stop"); + let root = directory.path(); + let request = NodeSetupRequest { + version: 1, + discovery_id: NodeIdentity::load_or_create(root).unwrap().uuid().to_string(), + operation_id: uuid::Uuid::new_v4().to_string(), + choices: NodeSetupChoices::default(), + }; + let authority = node_setup::TestAuthority::start(root, &request).await; + assert!(authority.registry.nodes[0].confirmed); + std::fs::write( + root.join("accepted-node.json"), + serde_json::to_vec(&serde_json::json!({ + "node_id":authority.binding.node_id,"bootstrap":authority.binding.bootstrap,"prepared":true, + })) + .unwrap(), + ) + .unwrap(); + std::fs::set_permissions( + root.join("accepted-node.json"), + std::fs::Permissions::from_mode(0o600), + ) + .unwrap(); + let intent = NodeServiceIntent { + cluster_id: authority.binding.bootstrap.cluster_id.clone(), + operation_id: uuid::Uuid::new_v4().to_string(), + node_id: authority.binding.node_id, + service_id: "kv-2".into(), + kind: "kv".into(), + action: NodeServiceAction::Stop, + configuration: "node_id = 2\n".into(), + environment: BTreeMap::new(), + }; + authority + .kv + .put( + 0, + 0, + b"/deployment/services/2/kv-2", + &serde_json::to_vec(&intent).unwrap(), + None, + ) + .await + .unwrap(); + let mut worker = NodeServiceForTests::new_for_tests(profile(root)).await.unwrap(); + let pid = worker.start_kv_child_for_tests().await.unwrap(); + let first = worker.execute_for_tests(intent.clone()).await.unwrap(); + assert!(first["pid"].is_null()); + assert!( + !Path::new(&format!("/proc/{pid}")).exists(), + "the acknowledged stop must reap its owned child" + ); + drop(authority); + Self { + directory, + worker, + intent, + } + } + + pub fn state(&self) -> PathBuf { + self.directory.path().join("services/kv-2/execution.json") + } +} + +fn profile(root: &Path) -> DeploymentProfile { + let mut profile = + DeploymentProfile::load(Path::new(env!("CARGO_MANIFEST_DIR")).join("../oci-image/node-profile.toml")) + .unwrap(); + profile.paths.data_root = root.to_path_buf(); + profile.paths.log_root = root.join("log"); + std::fs::create_dir_all(&profile.paths.log_root).unwrap(); + let program = root.join("owned-kv"); + std::fs::write(&program, "#!/bin/sh\nexec sleep 60\n").unwrap(); + std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o700)).unwrap(); + profile + .services + .iter_mut() + .find(|s| s.id == "kv") + .unwrap() + .program = program; + profile +} diff --git a/container/crowdb-monitor/tests/common/service_replacement.rs b/container/crowdb-monitor/tests/common/service_replacement.rs new file mode 100644 index 000000000..becccf06c --- /dev/null +++ b/container/crowdb-monitor/tests/common/service_replacement.rs @@ -0,0 +1,100 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crate::node_setup; +use crowdb_monitor::{DeploymentProfile, NodeIdentity, NodeServiceForTests}; +use crowdb_protocol::mgmt::node::{ + setup::{NodeSetupChoices, NodeSetupRequest}, + NodeServiceAction, NodeServiceIntent, +}; +use serde_json::json; +use std::{collections::BTreeMap, os::unix::fs::PermissionsExt}; + +pub struct TestReplacement { + pub directory: crowdb_test_harness::test_dirs::TestDir, + pub authority: node_setup::TestAuthority, + pub worker: NodeServiceForTests, + pub intent: NodeServiceIntent, + pub original: u32, + pub health: tokio::task::JoinHandle<()>, + pub state: std::path::PathBuf, +} + +impl TestReplacement { + pub async fn new() -> Self { + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("service-replacement"); + let root = directory.path(); + let request = NodeSetupRequest { + version: 1, + discovery_id: NodeIdentity::load_or_create(root).unwrap().uuid().to_string(), + operation_id: uuid::Uuid::new_v4().to_string(), + choices: NodeSetupChoices::default(), + }; + let authority = node_setup::TestAuthority::start(root, &request).await; + assert_eq!(authority.registry.nodes[0].node_id, authority.binding.node_id); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let health = tokio::spawn(async move { + axum::serve( + listener, + axum::Router::new().route("/health", axum::routing::get(|| async { "healthy" })), + ) + .await + .unwrap(); + }); + let mut profile = DeploymentProfile::load( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../oci-image/node-profile.toml"), + ) + .unwrap(); + profile.paths.data_root = root.to_path_buf(); + profile.paths.log_root = root.join("log"); + std::fs::create_dir_all(&profile.paths.log_root).unwrap(); + let program = root.join("owned-kv"); + std::fs::write(&program, "#!/bin/sh\nexec sleep 60\n").unwrap(); + std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o700)).unwrap(); + let service = profile + .services + .iter_mut() + .find(|service| service.id == "kv") + .unwrap(); + service.program = program; + service.probe.target = format!("http://{address}/health"); + let intent = NodeServiceIntent { + cluster_id: authority.binding.bootstrap.cluster_id.clone(), + operation_id: uuid::Uuid::new_v4().to_string(), + node_id: authority.binding.node_id, + service_id: "kv-2".into(), + kind: "kv".into(), + action: NodeServiceAction::Start, + configuration: "node_id = 2\n".into(), + environment: BTreeMap::new(), + }; + authority + .kv + .put( + 0, + 0, + b"/deployment/services/2/kv-2", + &serde_json::to_vec(&intent).unwrap(), + None, + ) + .await + .unwrap(); + let mut worker = NodeServiceForTests::new_for_tests(profile).await.unwrap(); + let original = worker.start_kv_child_for_tests().await.unwrap(); + let state = root.join("services/kv-2/execution.json"); + std::fs::create_dir_all(state.parent().unwrap()).unwrap(); + let record = json!({"intent":intent,"bootstrap":authority.binding.bootstrap,"complete":true,"pid":original,"attempts":0,"error":null}); + std::fs::write(&state, serde_json::to_vec(&record).unwrap()).unwrap(); + std::fs::set_permissions(&state, std::fs::Permissions::from_mode(0o600)).unwrap(); + Self { + directory, + authority, + worker, + intent, + original, + health, + state, + } + } +} diff --git a/container/crowdb-monitor/tests/iceberg_command_failure_test.rs b/container/crowdb-monitor/tests/iceberg_command_failure_test.rs new file mode 100644 index 000000000..671b7115d --- /dev/null +++ b/container/crowdb-monitor/tests/iceberg_command_failure_test.rs @@ -0,0 +1,90 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_monitor::{IcebergBootstrap, IcebergBootstrapError, ServerCredentials}; +use std::{fs, os::unix::fs::PermissionsExt, path::Path}; + +async fn failure(root: &Path, script: &str, credentials: &ServerCredentials) -> IcebergBootstrapError { + let program = root.join("management"); + fs::write(&program, script).unwrap(); + fs::set_permissions(&program, fs::Permissions::from_mode(0o700)).unwrap(); + IcebergBootstrap::ensure_console_catalog(root, &program, "127.0.0.1:1", credentials) + .await + .unwrap_err() +} + +#[tokio::test] +async fn command_failure_reports_action_exit_status_and_redacts_every_server_credential() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("iceberg-command-failure"); + let credentials = ServerCredentials::load_or_create(root.path()).unwrap(); + let script = r#"#!/bin/sh +printf 'catalog owner unavailable\n%s\n%s\n%s\n%s\n%s\n' \ + "$CROWDB_S3_MASTER_KEY" "$CROWDB_ICEBERG_READ_TOKEN" "$CROWDB_ICEBERG_WRITE_TOKEN" \ + "$CROWDB_ICEBERG_MANAGE_TOKEN" "$CROWDB_ICEBERG_CLEAR_TOKEN" >&2 +exit 17 +"#; + let error = failure(root.path(), script, &credentials).await; + let message = error.to_string(); + let IcebergBootstrapError::CommandExit { + action, + status, + diagnostic, + } = error + else { + panic!("expected a command exit diagnostic: {message}"); + }; + assert_eq!(action, "inspect"); + assert_eq!(status, "exit status: 17"); + assert!(diagnostic.contains("catalog owner unavailable")); + assert!(diagnostic.contains("")); + assert!(!diagnostic.chars().any(char::is_control)); + for line in credentials.server_env().lines() { + let (_, value) = line.split_once('=').unwrap(); + assert!(!message.contains(value), "a server credential escaped redaction"); + } +} + +#[tokio::test] +async fn oversized_multibyte_stderr_is_bounded_without_splitting_utf8() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("iceberg-command-bound"); + let credentials = ServerCredentials::load_or_create(root.path()).unwrap(); + let error = failure( + root.path(), + "#!/usr/bin/env python3\nimport sys\nsys.stderr.write('界' * 4096)\nsys.exit(3)\n", + &credentials, + ) + .await; + let IcebergBootstrapError::CommandExit { diagnostic, .. } = error else { + panic!("expected a command exit diagnostic"); + }; + assert_eq!(diagnostic, "界".repeat(1024)); +} + +#[tokio::test] +async fn a_signal_exit_remains_distinguishable_from_an_application_error() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("iceberg-command-signal"); + let credentials = ServerCredentials::load_or_create(root.path()).unwrap(); + let error = failure(root.path(), "#!/bin/sh\nkill -TERM $$\n", &credentials).await; + let IcebergBootstrapError::CommandExit { + status, diagnostic, .. + } = error + else { + panic!("expected a command exit diagnostic"); + }; + assert!(status.contains("signal: 15"), "{status}"); + assert_eq!(diagnostic, "no stderr was produced"); +} + +#[tokio::test] +async fn a_credential_crossing_the_diagnostic_bound_is_redacted_before_truncation() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("iceberg-command-boundary"); + let credentials = ServerCredentials::load_or_create(root.path()).unwrap(); + let padding = "x".repeat(1020); + let script = + format!("#!/bin/sh\nprintf '%s%s' '{padding}' \"$CROWDB_ICEBERG_MANAGE_TOKEN\" >&2\nexit 1\n"); + let error = failure(root.path(), &script, &credentials).await; + let IcebergBootstrapError::CommandExit { diagnostic, .. } = error else { + panic!("expected a command exit diagnostic"); + }; + assert_eq!(diagnostic, format!("{padding} (DeploymentProfile, PreparedBootstrap, NodeServiceCre "server": [{"id": "kv-1", "node_id": 1, "url": "http://127.0.0.1:10000", "rpc_url": "127.0.0.1:10100"}] })).unwrap(); let node = NodeRecord { + web_port: 9090, discovery_id: identity.uuid().to_string(), node_id: 1, rack_id: 1, diff --git a/container/crowdb-monitor/tests/kv_resume_test.rs b/container/crowdb-monitor/tests/kv_resume_test.rs index 077df71f2..875d03556 100644 --- a/container/crowdb-monitor/tests/kv_resume_test.rs +++ b/container/crowdb-monitor/tests/kv_resume_test.rs @@ -4,7 +4,10 @@ #[path = "common/member_trust.rs"] mod trust; -use crowdb_monitor::{accept_member_trust_for_tests, DeploymentProfile, NodeServiceForTests, StatusStore}; +use crowdb_monitor::{ + accept_member_trust_for_tests, accept_node_policy, DeploymentProfile, NodeServiceForTests, StatusStore, +}; +use crowdb_protocol::mgmt::node::deployment::NodeDeploymentOverrides; use std::{ os::unix::fs::PermissionsExt, sync::atomic::{AtomicBool, Ordering}, @@ -16,6 +19,15 @@ use std::{ async fn retained_quorum_startup_does_not_block_discovery_or_confuse_liveness_with_health() { let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("kv-resume"); let root = directory.path(); + accept_node_policy( + root, + &NodeDeploymentOverrides::default(), + &NodeDeploymentOverrides { + initial_voters: Some(3), + ..Default::default() + }, + ) + .unwrap(); let trust = trust::fixture(root); accept_member_trust_for_tests(root, &trust).unwrap(); let mut profile = DeploymentProfile::load( diff --git a/container/crowdb-monitor/tests/member_discovery_test.rs b/container/crowdb-monitor/tests/member_discovery_test.rs new file mode 100644 index 000000000..b9b52e71d --- /dev/null +++ b/container/crowdb-monitor/tests/member_discovery_test.rs @@ -0,0 +1,118 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/member_trust.rs"] +mod trust; + +#[tokio::test] +async fn stale_authority_contacts_cannot_starve_publication_through_recovered_routes() { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::time::{Duration, Instant}; + let published = AtomicBool::new(false); + let start = Instant::now(); + crowdb_monitor::recovery_authority_work_for_tests( + async { + tokio::time::sleep(Duration::from_millis(1)).await; + published.store(true, Ordering::Release); + Ok(()) + }, + std::future::pending(), + ) + .await; + assert!( + published.load(Ordering::Acquire), + "old unavailable authority contacts consumed the publication budget" + ); + assert!( + start.elapsed() < Duration::from_secs(4), + "confirmation and publication retain one shared three-second budget" + ); +} + +#[test] +fn pinned_retained_contacts_are_probed_when_multicast_discovery_is_empty() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("member-recovery-contacts"); + let trust = trust::fixture(root.path()); + let operation = crowdb_console_shared::deployment::PreparedBootstrap::load( + &root.path().join("prepared-bootstrap.json"), + ) + .unwrap(); + let mut config = operation.intent.to_config(); + for node in &mut config.nodes { + node.ssh_port = 2200 + u16::try_from(node.id).unwrap(); + node.ssh_password = Some("discarded-test-password".into()); + } + let mut foreign = config.nodes[0].clone(); + foreign.id = 99; + foreign.host = "192.0.2.99".into(); + config.nodes.push(foreign); + let probes = crowdb_monitor::member_recovery_destinations_for_tests( + root.path(), + &config, + &trust, + crowdb_protocol::mgmt::node::CandidateSnapshot::default(), + ) + .unwrap(); + assert_eq!(probes.iter().map(|node| node.id).collect::>(), [1, 2, 3]); + for node in probes { + assert_eq!(node.host, format!("192.0.2.{}", node.id)); + assert_eq!(node.ssh_port, 2200 + u16::try_from(node.id).unwrap()); + assert!(node.ssh_password.is_none()); + assert_eq!( + node.ssh_key, + Some(root.path().join("ssh/id_ed25519").to_string_lossy().into_owned()) + ); + } +} + +#[test] +fn retained_contacts_do_not_crowd_a_changed_address_out_of_the_probe_budget() { + use crowdb_protocol::mgmt::node::{CandidateNode, CandidateSnapshot, CandidateState, NodeAdvertisement}; + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("member-recovery-probe-budget"); + let mut trust = trust::fixture(root.path()); + let operation = crowdb_console_shared::deployment::PreparedBootstrap::load( + &root.path().join("prepared-bootstrap.json"), + ) + .unwrap(); + let mut config = operation.intent.to_config(); + let template_node = config.nodes[0].clone(); + let template_member = trust.members[0].clone(); + config.nodes.clear(); + trust.members.clear(); + for id in 1_u8..=64 { + let mut node = template_node.clone(); + node.id = u64::from(id); + node.host = format!("192.0.2.{id}"); + config.nodes.push(node); + let mut member = template_member.clone(); + member.node_id = u64::from(id); + member.discovery_id = uuid::Uuid::new_v4().to_string(); + member.admission_operation_id = uuid::Uuid::new_v4().to_string(); + member.host_key = trust::key(id); + trust.members.push(member); + } + trust.validate().unwrap(); + let snapshot = CandidateSnapshot { + nodes: vec![CandidateNode { + advertisement: NodeAdvertisement { + discovery_id: trust.members[63].discovery_id.clone(), + protocol_version: 1, + monitor_endpoints: vec!["http://203.0.113.64:9095".into()], + cluster_id: Some(trust.bootstrap.cluster_id.clone()), + formation: None, + }, + state: CandidateState::SameCluster, + }], + discovery_diagnostics: Vec::new(), + }; + let probes = + crowdb_monitor::member_recovery_destinations_for_tests(root.path(), &config, &trust, snapshot) + .unwrap(); + assert_eq!(probes.len(), 64); + assert!( + probes + .iter() + .any(|node| node.id == 64 && node.host == "203.0.113.64"), + "a full retained-address cache must not suppress a member's discovered new address" + ); +} diff --git a/container/crowdb-monitor/tests/member_trust_test.rs b/container/crowdb-monitor/tests/member_trust_test.rs index df93c053a..bc21fc72b 100644 --- a/container/crowdb-monitor/tests/member_trust_test.rs +++ b/container/crowdb-monitor/tests/member_trust_test.rs @@ -13,7 +13,7 @@ mod run_root; use run_root::TestRunRoot; #[test] -fn duplicate_authenticated_incarnations_remove_the_route_in_any_order() { +fn restarted_authenticated_member_keeps_the_current_route() { use crowdb_protocol::mgmt::node::recovery::MemberEndpoint; let first = MemberEndpoint { node_id: 1, @@ -24,11 +24,22 @@ fn duplicate_authenticated_incarnations_remove_the_route_in_any_order() { let mut cloned = first.clone(); cloned.boot_id = uuid::Uuid::new_v4().to_string(); cloned.addresses = vec!["192.0.2.2".parse().unwrap()]; - for records in [vec![first.clone(), cloned.clone()], vec![cloned, first.clone()]] { + for records in [ + vec![first.clone(), cloned.clone()], + vec![cloned.clone(), first.clone()], + ] { let (members, conflicts) = crowdb_monitor::unique_member_endpoints_for_tests(records); - assert!(members.is_empty()); - assert_eq!(conflicts, vec![first.discovery_id.clone()]); + assert_eq!(members.len(), 1); + assert_eq!(members[0].discovery_id, first.discovery_id); + assert!(conflicts.is_empty()); } + let mut foreign = first.clone(); + foreign.discovery_id = uuid::Uuid::new_v4().to_string(); + let (members, conflicts) = + crowdb_monitor::unique_member_endpoints_for_tests(vec![first.clone(), foreign]); + assert!(members.is_empty()); + assert_eq!(conflicts.len(), 2); + assert!(conflicts.contains(&first.discovery_id)); let (members, conflicts) = crowdb_monitor::unique_member_endpoints_for_tests(vec![first.clone(), first.clone()]); assert_eq!(members, vec![first]); @@ -364,3 +375,51 @@ fn a_new_admission_keeps_the_cancelled_operations_pins_and_fence() { "the cancelled operation remains fenced" ); } + +#[test] +fn recovered_contacts_preserve_member_specific_listener_ports() { + use crowdb_protocol::mgmt::node::recovery::{MemberEndpoint, MemberEndpoints}; + let dir = tempdir_in_test_data("monitor-offset-recovery-contacts"); + let trust = trust::fixture(dir.path()); + let operation = crowdb_console_shared::deployment::PreparedBootstrap::load( + &dir.path().join("prepared-bootstrap.json"), + ) + .unwrap(); + let mut config = operation.intent.to_config(); + config.servers[1].url = "http://192.0.2.2:10001".into(); + config.servers[1].rpc_url = Some("192.0.2.2:10101".into()); + let endpoints = MemberEndpoints { + version: 1, + bootstrap: trust.bootstrap, + observer_boot_id: uuid::Uuid::new_v4().to_string(), + observed_at_unix_ms: 1, + conflicts: vec![], + members: trust + .members + .into_iter() + .map(|member| MemberEndpoint { + discovery_id: member.discovery_id, + node_id: member.node_id, + boot_id: uuid::Uuid::new_v4().to_string(), + addresses: vec![if member.node_id == 3 { + "2001:db8::3" + } else { + "192.0.2.10" + } + .parse() + .unwrap()], + }) + .collect(), + }; + let seeds = crowdb_monitor::apply_member_endpoints_for_tests(&mut config, &endpoints).unwrap(); + assert_eq!( + seeds, + [ + "http://192.0.2.10:10000", + "http://192.0.2.10:10001", + "http://[2001:db8::3]:10000" + ] + ); + assert_eq!(config.servers[1].rpc_url.as_deref(), Some("192.0.2.10:10101")); + assert_eq!(config.servers[2].rpc_url.as_deref(), Some("[2001:db8::3]:10100")); +} diff --git a/container/crowdb-monitor/tests/node_cleanup_test.rs b/container/crowdb-monitor/tests/node_cleanup_test.rs index d878c5e7c..14e37871b 100644 --- a/container/crowdb-monitor/tests/node_cleanup_test.rs +++ b/container/crowdb-monitor/tests/node_cleanup_test.rs @@ -1,7 +1,11 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use crowdb_monitor::{cleanup_node_for_tests, DeploymentProfile, ProcessManager}; +use crowdb_monitor::{accept_node_policy, cleanup_node_for_tests, DeploymentProfile, ProcessManager}; +use crowdb_protocol::mgmt::node::{ + deployment::{NodeDeploymentOverrides, NodeDeploymentPolicy}, + setup::SetupServiceKind, +}; use crowdb_protocol::mgmt::SystemBootstrapIdentity; use crowdb_test_harness::test_dirs::tempdir_in_test_data; use std::{os::unix::fs::PermissionsExt, path::Path}; @@ -11,6 +15,19 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; async fn retired_cleanup_restarts_its_local_worker_after_a_failed_deletion() { let directory = tempdir_in_test_data("monitor-retired-cleanup"); let root = directory.path(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let mut services = NodeDeploymentPolicy::default().setup.services; + services + .iter_mut() + .find(|service| service.kind == SetupServiceKind::PaxosKv) + .unwrap() + .listeners + .http_port = Some(listener.local_addr().unwrap().port()); + let defaults = NodeDeploymentOverrides { + services: Some(services), + ..Default::default() + }; + accept_node_policy(root, &defaults, &defaults).unwrap(); let bootstrap = SystemBootstrapIdentity { cluster_id: uuid::Uuid::new_v4().to_string(), operation_id: uuid::Uuid::new_v4().to_string(), @@ -62,7 +79,6 @@ async fn retired_cleanup_restarts_its_local_worker_after_a_failed_deletion() { assert!(!retired.exists()); assert!(!manager.owns("kv")); - let listener = tokio::net::TcpListener::bind("127.0.0.1:10000").await.unwrap(); let endpoint = tokio::spawn(serve_deletions(listener, bootstrap.clone(), retired.clone())); assert!( cleanup_node_for_tests(&profile, &mut manager, bootstrap.clone(), true) diff --git a/container/crowdb-monitor/tests/node_health_authority_test.rs b/container/crowdb-monitor/tests/node_health_authority_test.rs new file mode 100644 index 000000000..45c39438d --- /dev/null +++ b/container/crowdb-monitor/tests/node_health_authority_test.rs @@ -0,0 +1,40 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/node_health.rs"] +mod fixture; + +use crowdb_monitor::MonitorPhase; +use fixture::TestHealth; +use std::{sync::atomic::Ordering, time::Duration}; + +#[tokio::test] +async fn slow_service_authority_does_not_freeze_current_local_health() { + let mut fixture = TestHealth::new().await; + tokio::time::timeout(Duration::from_secs(1), fixture.worker.tick_for_tests()) + .await + .expect("service authority must not block local health publication") + .unwrap(); + tokio::time::timeout(Duration::from_secs(1), fixture.entered.notified()) + .await + .expect("the production authority read was actually started"); + let healthy = fixture.snapshot(); + assert_eq!(healthy.phase, MonitorPhase::Ready); + assert_eq!(healthy.services.len(), 2); + assert!(healthy.services.values().all(|service| service.healthy)); + + fixture.ready.store(false, Ordering::Release); + tokio::time::timeout(Duration::from_secs(1), fixture.worker.tick_for_tests()) + .await + .expect("an in-flight authority read must not hide changed local health") + .unwrap(); + let unavailable = fixture.snapshot(); + assert_eq!(unavailable.phase, MonitorPhase::Restarting); + assert!(unavailable.services.values().all(|service| !service.healthy)); + assert!(unavailable.revision > healthy.revision); + for (id, service) in &unavailable.services { + assert_eq!(service.pid, healthy.services[id].pid); + assert_eq!(service.restart_attempts, 0); + } + fixture.worker.stop_kv_for_tests().await.unwrap(); +} diff --git a/container/crowdb-monitor/tests/node_managed_health_test.rs b/container/crowdb-monitor/tests/node_managed_health_test.rs new file mode 100644 index 000000000..fd90b7a26 --- /dev/null +++ b/container/crowdb-monitor/tests/node_managed_health_test.rs @@ -0,0 +1,73 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/managed_health.rs"] +mod fixture; +#[path = "common/node_setup.rs"] +mod node_setup; + +use fixture::TestManagedHealth; +use std::{ + sync::atomic::Ordering, + time::{Duration, Instant}, +}; + +#[tokio::test] +async fn background_recovery_yields_while_a_new_managed_process_is_not_ready() { + let mut test = TestManagedHealth::new().await; + test.worker.stop_child_for_tests("chunkdb-2").await.unwrap(); + test.ready.store(false, Ordering::Release); + let started = Instant::now(); + let result = test.worker.reconcile_for_tests().await; + let elapsed = started.elapsed(); + let pending = test.execution(); + test.worker.tick_for_tests().await.unwrap(); + let unavailable = crowdb_monitor::StatusStore::open(&test.profile.paths.run_root) + .unwrap() + .read(Duration::from_secs(3)) + .unwrap(); + test.ready.store(true, Ordering::Release); + test.worker.reconcile_for_tests().await.unwrap(); + let recovered = test.execution(); + test.stop().await; + result.unwrap(); + assert!( + elapsed < Duration::from_secs(3), + "background recovery held health publication for {elapsed:?}" + ); + assert!(!pending["complete"].as_bool().unwrap()); + assert_eq!(pending["attempts"], 1); + assert!(!unavailable.services["chunkdb-2"].healthy); + assert_eq!( + unavailable.services["chunkdb-2"].pid, + pending["pid"].as_u64().map(|pid| u32::try_from(pid).unwrap()) + ); + assert!(recovered["complete"].as_bool().unwrap()); + assert_eq!(recovered["pid"], pending["pid"]); + assert_eq!(recovered["intent"]["operation_id"], test.intent.operation_id); +} + +#[tokio::test] +async fn slow_probes_share_one_health_pass_and_do_not_hide_real_failures() { + let mut test = TestManagedHealth::new().await; + test.slow.store(true, Ordering::Release); + let started = Instant::now(); + test.worker.tick_for_tests().await.unwrap(); + let elapsed = started.elapsed(); + let store = crowdb_monitor::StatusStore::open(&test.profile.paths.run_root).unwrap(); + let unavailable = store.read(Duration::from_secs(3)).unwrap(); + test.slow.store(false, Ordering::Release); + test.worker.tick_for_tests().await.unwrap(); + let recovered = store.read(Duration::from_secs(3)).unwrap(); + test.stop().await; + assert!( + elapsed < Duration::from_secs(2), + "probe deadlines accumulated: {elapsed:?}" + ); + assert_eq!(unavailable.services.len(), 3); + assert!(unavailable.services.values().all(|service| !service.healthy)); + assert!(recovered.services.values().all(|service| service.healthy)); + for (id, service) in &unavailable.services { + assert_eq!(service.pid, recovered.services[id].pid); + } +} diff --git a/container/crowdb-monitor/tests/node_policy_test.rs b/container/crowdb-monitor/tests/node_policy_test.rs index eeca7391d..0184deb97 100644 --- a/container/crowdb-monitor/tests/node_policy_test.rs +++ b/container/crowdb-monitor/tests/node_policy_test.rs @@ -10,6 +10,57 @@ use crowdb_protocol::mgmt::node::deployment::NodeDeploymentOverrides; #[path = "common/run_root.rs"] mod run_root; +#[test] +fn explicit_file_listener_ports_are_shifted_once_across_repeated_acceptance() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-policy-file-offset"); + let omitted = NodeDeploymentOverrides::default(); + let mut file = NodeDeploymentOverrides { + services: Some( + crowdb_protocol::mgmt::node::deployment::NodeDeploymentPolicy::default() + .setup + .services, + ), + ..Default::default() + }; + let accepted = + crowdb_monitor::accept_node_policy_with_port_offset(root.path(), &file, &omitted, 1).unwrap(); + let bytes = fs::read(root.path().join("node-policy.json")).unwrap(); + assert_eq!( + crowdb_monitor::accept_node_policy_with_port_offset(root.path(), &file, &omitted, 1).unwrap(), + accepted + ); + file.services.as_mut().unwrap()[0].listeners.rpc_port = Some(10102); + assert!(crowdb_monitor::accept_node_policy_with_port_offset(root.path(), &file, &omitted, 1).is_err()); + assert_eq!(fs::read(root.path().join("node-policy.json")).unwrap(), bytes); +} + +#[test] +fn host_network_policy_shifts_every_diskdb_listener_once_and_keeps_instances_disjoint() { + let omitted = NodeDeploymentOverrides::default(); + let mut listeners = std::collections::BTreeSet::new(); + for offset in 0..3 { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-policy-offset"); + let policy = + crowdb_monitor::accept_node_policy_with_port_offset(root.path(), &omitted, &omitted, offset) + .unwrap(); + let diskdb = &policy.setup.services[1].listeners; + assert_eq!(diskdb.listen_port, Some(11000 + offset)); + assert_eq!(diskdb.http_port, Some(11100 + offset)); + assert_eq!(diskdb.rpc_port, Some(11200 + offset)); + for port in [diskdb.listen_port, diskdb.http_port, diskdb.rpc_port] + .into_iter() + .flatten() + { + assert!(listeners.insert(port)); + } + assert_eq!( + crowdb_monitor::accept_node_policy_with_port_offset(root.path(), &omitted, &omitted, offset,) + .unwrap(), + policy + ); + } +} + #[test] fn accepted_policy_survives_restart_without_defaults_or_identity_effects() { let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("accepted-node-policy"); diff --git a/container/crowdb-monitor/tests/node_process_operation_test.rs b/container/crowdb-monitor/tests/node_process_operation_test.rs new file mode 100644 index 000000000..9e8b8a6e9 --- /dev/null +++ b/container/crowdb-monitor/tests/node_process_operation_test.rs @@ -0,0 +1,95 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/runtime_observation.rs"] +mod fixture; +use crowdb_monitor::NodeServiceForTests; +use crowdb_protocol::mgmt::{node::NodeBinding, SystemBootstrapIdentity}; +use serde_json::{json, Value}; +use std::{os::unix::fs::PermissionsExt, time::Duration}; + +#[tokio::test] +async fn process_operations_and_failures_require_the_actual_pid_and_current_bound_execution() { + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("process-operation"); + let mut test = fixture::TestObservation::start(directory.path()).await; + let binding = NodeBinding { + node_id: test.intent.node_id, + bootstrap: SystemBootstrapIdentity { + cluster_id: test.intent.cluster_id.clone(), + operation_id: "bootstrap".into(), + configuration_digest: "a".repeat(64), + }, + management_seeds: vec!["http://127.0.0.1:1".into()], + }; + std::fs::write( + directory.path().join("node-binding.json"), + serde_json::to_vec(&binding).unwrap(), + ) + .unwrap(); + std::fs::set_permissions( + directory.path().join("node-binding.json"), + std::fs::Permissions::from_mode(0o600), + ) + .unwrap(); + let state = directory.path().join("services/chunkdb-1/execution.json"); + let mut record: Value = serde_json::from_slice(&std::fs::read(&state).unwrap()).unwrap(); + record["bootstrap"] = json!(binding.bootstrap); + record["attempts"] = json!(5); + record["error"] = json!("retained diagnostic"); + std::fs::write(&state, serde_json::to_vec(&record).unwrap()).unwrap(); + let pid = test.manager.pid("chunkdb-1").unwrap(); + assert_eq!( + test.store.read(Duration::from_secs(3)).unwrap().services["chunkdb-1"].pid, + Some(pid) + ); + let read = |manager: &mut crowdb_monitor::ProcessManager| { + NodeServiceForTests::processes_for_tests( + directory.path(), + manager, + &binding.bootstrap.cluster_id, + binding.node_id, + ) + .unwrap() + }; + let observed = read(&mut test.manager); + assert_eq!(observed["bootstrap"], json!(binding.bootstrap)); + assert_eq!(observed["processes"]["chunkdb-1"], pid); + assert_eq!( + observed["process_operations"]["chunkdb-1"], + test.intent.operation_id + ); + assert_eq!(observed["service_failures"]["chunkdb-1"]["exhausted"], false); + for fault in ["legacy", "bootstrap", "cluster", "node", "pid"] { + let mut invalid = record.clone(); + match fault { + "legacy" => { + invalid.as_object_mut().unwrap().remove("bootstrap"); + } + "bootstrap" => invalid["bootstrap"]["operation_id"] = json!("old-bootstrap"), + "cluster" => invalid["intent"]["cluster_id"] = json!("other-cluster"), + "node" => invalid["intent"]["node_id"] = json!(99), + "pid" => invalid["pid"] = json!(pid + 100_000), + _ => unreachable!(), + } + std::fs::write(&state, serde_json::to_vec(&invalid).unwrap()).unwrap(); + let observed = read(&mut test.manager); + assert_eq!( + observed["processes"]["chunkdb-1"], pid, + "actual children remain observable: {fault}" + ); + assert!( + observed["process_operations"]["chunkdb-1"].is_null(), + "foreign execution labeled the live PID: {fault}" + ); + if fault != "pid" { + assert!( + observed["service_failures"]["chunkdb-1"].is_null(), + "foreign execution exhausted the current start: {fault}" + ); + } + } + test.manager + .stop("chunkdb-1", Duration::from_secs(1)) + .await + .unwrap(); +} diff --git a/container/crowdb-monitor/tests/node_run_inputs_test.rs b/container/crowdb-monitor/tests/node_run_inputs_test.rs index f7e0ba356..d3f62fc6b 100644 --- a/container/crowdb-monitor/tests/node_run_inputs_test.rs +++ b/container/crowdb-monitor/tests/node_run_inputs_test.rs @@ -36,6 +36,56 @@ fn command(profile: &Path) -> Command { command } +#[test] +fn container_preflight_reuses_explicit_file_ports_with_a_nonzero_offset() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-cli-file-offset"); + let profile = profile(root.path()); + let config = root.path().join("node.toml"); + let inputs = NodeDeploymentOverrides { + services: Some( + crowdb_protocol::mgmt::node::deployment::NodeDeploymentPolicy::default() + .setup + .services, + ), + ..Default::default() + }; + fs::write(&config, toml::to_string(&inputs).unwrap()).unwrap(); + let invoke = || { + let mut accept = Command::new(env!("CARGO_BIN_EXE_crowdb-monitor")); + for name in [ + "CROWDB_NODE_POLICY_FILE", + "CROWDB_DEPLOYMENT_MODE", + "CROWDB_CLUSTER", + "CROWDB_INITIAL_VOTERS", + "CROWDB_PHYSICAL_HOST_ID", + "CROWDB_S3_PUBLIC_URI", + "CROWDB_ICEBERG_PUBLIC_URI", + "CROWDB_DATASET_PUBLIC_URI", + "CROWDB_DISCOVERY_SEEDS", + ] { + accept.env_remove(name); + } + accept.args(["accept-policy", "--profile"]).arg(&profile); + accept + .args(["--deployment-config"]) + .arg(&config) + .env("CROWDB_PORT_OFFSET", "1"); + accept.output().unwrap() + }; + for _ in 0..2 { + let output = invoke(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + } + let accepted = crowdb_monitor::read_node_policy(&root.path().join("install/data")) + .unwrap() + .unwrap(); + assert_eq!(accepted.setup.services[0].listeners.rpc_port, Some(10101)); +} + #[test] fn public_uri_flags_override_environment_before_acceptance_and_changed_restarts_fail() { let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("public-origin"); @@ -89,7 +139,7 @@ fn invalid_voter_override_fails_before_node_identity_and_storage_effects() { .output() .unwrap(); assert!(!output.status.success()); - assert!(String::from_utf8_lossy(&output.stderr).contains("initial voters must be one or three")); + assert!(String::from_utf8_lossy(&output.stderr).contains("initial voters must be one, three or five")); assert_eq!(fs::read_dir(root.path().join("install/data")).unwrap().count(), 0); } diff --git a/container/crowdb-monitor/tests/node_service_dependencies_test.rs b/container/crowdb-monitor/tests/node_service_dependencies_test.rs index 7b2f6a8d3..a8184d96e 100644 --- a/container/crowdb-monitor/tests/node_service_dependencies_test.rs +++ b/container/crowdb-monitor/tests/node_service_dependencies_test.rs @@ -193,7 +193,7 @@ async fn access_readiness_follows_catalog_redirects_within_its_probe_deadline() }; let authority = node_setup::TestAuthority::start(root.path(), &request).await; let (rpc, requests) = redirecting_rpc(); - publish_catalog(&authority.kv, rpc.port()).await; + publish_catalog(&authority.kv, rpc.port(), None).await; let mut profile = DeploymentProfile::load(Path::new(env!("CARGO_MANIFEST_DIR")).join("../oci-image/node-profile.toml")) .unwrap(); @@ -217,9 +217,34 @@ async fn access_readiness_follows_catalog_redirects_within_its_probe_deadline() environment: BTreeMap::new(), }; let result = NodeServiceForTests::dependencies_ready_for_tests(&profile, &intent, &[]).await; - rpc.stop(); result.unwrap(); assert_eq!(requests.load(Ordering::Relaxed), 2); + let (second_rpc, second_requests) = redirecting_rpc(); + publish_catalog( + &authority.kv, + rpc.port(), + Some(u16::try_from(second_rpc.port()).unwrap()), + ) + .await; + NodeServiceForTests::dependencies_ready_for_tests(&profile, &intent, &[]) + .await + .unwrap(); + assert_eq!(second_requests.load(Ordering::Relaxed), 2); + let unavailable = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + publish_catalog( + &authority.kv, + rpc.port(), + Some(unavailable.local_addr().unwrap().port()), + ) + .await; + let result = NodeServiceForTests::dependencies_ready_for_tests(&profile, &intent, &[]).await; + rpc.stop(); + second_rpc.stop(); + assert!( + result.is_err(), + "a healthy first owner must not hide an unavailable second owner" + ); + assert!(!root.path().join("services/access-2/execution.json").exists()); } fn redirecting_rpc() -> (crowdb_rpc_ffi::RpcServer, Arc) { @@ -258,7 +283,7 @@ fn redirecting_rpc() -> (crowdb_rpc_ffi::RpcServer, Arc) { (rpc, requests) } -async fn publish_catalog(kv: &crowdb_kv_client::CrowdbKvClient, port: i32) { +async fn publish_catalog(kv: &crowdb_kv_client::CrowdbKvClient, port: i32, second: Option) { let mut page = ChunkKvRangeCatalogPage { generation: 1, page_index: 0, @@ -283,6 +308,20 @@ async fn publish_catalog(kv: &crowdb_kv_client::CrowdbKvClient, port: i32) { }, }], }; + if let Some(port) = second { + page.entries[0].range.end = Some(vec![128]); + let mut entry = page.entries[0].clone(); + entry.partition_id.low = 2; + entry.artifact.tree_id = 2; + entry.artifact.stream_name.low = 2; + entry.range = KeyRange { + start: vec![128], + end: None, + }; + entry.owner.instance_id = 3; + entry.owner.rpc_endpoint = format!("127.0.0.1:{port}"); + page.entries.push(entry); + } page.seal().unwrap(); let mut head = ChunkKvRangeCatalogHead { generation: 1, diff --git a/container/crowdb-monitor/tests/node_service_readiness_test.rs b/container/crowdb-monitor/tests/node_service_readiness_test.rs index 2a84e6cdc..44be04856 100644 --- a/container/crowdb-monitor/tests/node_service_readiness_test.rs +++ b/container/crowdb-monitor/tests/node_service_readiness_test.rs @@ -220,3 +220,184 @@ async fn recovery_waits_for_live_dependencies_without_consuming_launches() { server.abort(); } } + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn completed_service_preserves_live_recovery_and_restarts_failed_health() { + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("service-health-restart"); + let root = directory.path().join("data"); + std::fs::create_dir_all(&root).unwrap(); + let request = NodeSetupRequest { + version: 1, + discovery_id: NodeIdentity::load_or_create(&root).unwrap().uuid().to_string(), + operation_id: uuid::Uuid::new_v4().to_string(), + choices: crowdb_protocol::mgmt::node::setup::NodeSetupChoices::default(), + }; + let authority = node_setup::TestAuthority::start(&root, &request).await; + let mut profile = DeploymentProfile::load( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../oci-image/node-profile.toml"), + ) + .unwrap(); + profile.paths.data_root = root.clone(); + profile.paths.install_root = directory.path().to_path_buf(); + profile.paths.bin_root = directory.path().join("bin"); + profile.paths.template_root = directory.path().join("templates"); + profile.paths.log_root = root.join("log"); + profile.paths.run_root = directory.path().join("run"); + std::fs::create_dir_all(&profile.paths.bin_root).unwrap(); + std::fs::create_dir_all(&profile.paths.template_root).unwrap(); + std::fs::create_dir_all(&profile.paths.log_root).unwrap(); + std::fs::create_dir_all(&profile.paths.run_root).unwrap(); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + drop(listener); + let unhealthy = root.join("unhealthy"); + let invocations = root.join("invocations"); + let listening = root.join("listening"); + let program = profile.paths.bin_root.join("managed-service"); + std::fs::write( + &program, + format!( + r#"#!/bin/sh +count=0 +if [ -f '{invocations}' ]; then count=$(cat '{invocations}'); fi +count=$((count + 1)) +echo "$count" > '{invocations}' +health_file='' +if [ "$count" -eq 1 ]; then health_file='{unhealthy}'; fi +exec python3 - "{port}" "$health_file" <<'PY' +from http.server import BaseHTTPRequestHandler, HTTPServer +import os +import sys + +port = int(sys.argv[1]) +health_file = sys.argv[2] + +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + status = 200 + if health_file and os.path.exists(health_file): + with open(health_file) as marker: + readiness_only = marker.read() == "readiness-only" + if self.path == "/ready" or not readiness_only: + status = 503 + self.send_response(status) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *_args): + pass + +server = HTTPServer(("127.0.0.1", port), Handler) +with open("{invocations}") as marker: + invocation = marker.read().strip() +with open("{listening}", "w") as marker: + marker.write(invocation) +server.serve_forever() +PY +"#, + invocations = invocations.display(), + unhealthy = unhealthy.display(), + port = port, + listening = listening.display(), + ), + ) + .unwrap(); + std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o700)).unwrap(); + for service in &mut profile.services { + service.program = profile.paths.bin_root.join("unused-service"); + } + let child = profile + .services + .iter_mut() + .find(|service| service.id == "chunkdb") + .unwrap(); + child.program = program; + child.dependencies = vec!["kv".into()]; + child.probe.failure_threshold = 2; + profile.validate().unwrap(); + + let intent = NodeServiceIntent { + cluster_id: authority.binding.bootstrap.cluster_id.clone(), + operation_id: uuid::Uuid::new_v4().to_string(), + node_id: authority.binding.node_id, + service_id: "chunkdb-2".into(), + kind: "chunkdb".into(), + action: NodeServiceAction::Start, + configuration: format!("[server]\nhttp_listen_addr = '127.0.0.1:{port}'\n"), + environment: std::collections::BTreeMap::default(), + }; + let key = b"/deployment/services/2/chunkdb-2"; + authority + .kv + .put(0, 0, key, &serde_json::to_vec(&intent).unwrap(), None) + .await + .unwrap(); + + let mut worker = NodeServiceForTests::new_for_tests(profile).await.unwrap(); + let first = worker.execute_for_tests(intent.clone()).await.unwrap(); + let first_pid = first["pid"].as_u64().unwrap(); + std::fs::write(&unhealthy, b"readiness-only").unwrap(); + assert_eq!( + reqwest::get(format!("http://127.0.0.1:{port}/ready")) + .await + .unwrap() + .status(), + reqwest::StatusCode::SERVICE_UNAVAILABLE + ); + for _ in 0..3 { + worker.reconcile_for_tests().await.unwrap(); + } + let recovering: serde_json::Value = + serde_json::from_slice(&std::fs::read(root.join("services/chunkdb-2/execution.json")).unwrap()) + .unwrap(); + assert_eq!(recovering["pid"].as_u64().unwrap(), first_pid); + assert_eq!( + std::fs::read_to_string(&invocations).unwrap().trim(), + "1", + "readiness recovery must retain a responsive process" + ); + std::fs::write(&unhealthy, b"probe failure").unwrap(); + worker.reconcile_for_tests().await.unwrap(); + assert_eq!( + std::fs::read_to_string(&invocations).unwrap().trim(), + "1", + "one failed probe must not restart a live service" + ); + worker.reconcile_for_tests().await.unwrap(); + let state: serde_json::Value = + serde_json::from_slice(&std::fs::read(root.join("services/chunkdb-2/execution.json")).unwrap()) + .unwrap(); + assert_ne!(state["pid"].as_u64().unwrap(), first_pid); + // Background recovery yields while the new listener starts. Wait for the + // fixture's actual bind event, then observe the same child on its next pass. + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while std::fs::read_to_string(&listening).unwrap_or_default() != "2" { + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .expect("replacement fixture did not bind its listener"); + worker.reconcile_for_tests().await.unwrap(); + let ready: serde_json::Value = + serde_json::from_slice(&std::fs::read(root.join("services/chunkdb-2/execution.json")).unwrap()) + .unwrap(); + assert_eq!(ready["complete"], true); + assert_eq!(ready["pid"], state["pid"]); + assert_eq!( + std::fs::read_to_string(&invocations).unwrap().trim(), + "2", + "the threshold must restart the listener even while its child remains alive" + ); + + let mut stop = intent; + stop.operation_id = uuid::Uuid::new_v4().to_string(); + stop.action = NodeServiceAction::Stop; + authority + .kv + .put(0, 0, key, &serde_json::to_vec(&stop).unwrap(), None) + .await + .unwrap(); + worker.execute_for_tests(stop).await.unwrap(); +} diff --git a/container/crowdb-monitor/tests/node_service_receipt_test.rs b/container/crowdb-monitor/tests/node_service_receipt_test.rs new file mode 100644 index 000000000..f06e553ac --- /dev/null +++ b/container/crowdb-monitor/tests/node_service_receipt_test.rs @@ -0,0 +1,109 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/node_setup.rs"] +mod node_setup; +#[path = "common/service_receipt.rs"] +mod service_receipt; + +use service_receipt::TestStoppedKv; +use std::time::Duration; + +#[tokio::test] +async fn completed_kv_stop_can_report_its_receipt_after_authority_loss() { + let mut test = TestStoppedKv::new().await; + let before = std::fs::read(test.state()).unwrap(); + let receipt = tokio::time::timeout( + Duration::from_millis(200), + test.worker.execute_for_tests(test.intent.clone()), + ) + .await + .expect("reporting a completed stop must not wait for the authority it stopped") + .unwrap(); + assert_eq!(receipt["operation_id"], test.intent.operation_id); + assert!(receipt["pid"].is_null()); + assert_eq!( + std::fs::read(test.state()).unwrap(), + before, + "a receipt must not repeat execution" + ); +} + +#[tokio::test] +async fn incomplete_foreign_and_legacy_receipts_cannot_authorize_an_offline_operation() { + let mut test = TestStoppedKv::new().await; + let original: serde_json::Value = serde_json::from_slice(&std::fs::read(test.state()).unwrap()).unwrap(); + for fault in [ + "legacy", + "foreign-bootstrap", + "incomplete", + "pid", + "newer-operation", + ] { + let mut record = original.clone(); + match fault { + "legacy" => { + record.as_object_mut().unwrap().remove("bootstrap"); + } + "foreign-bootstrap" => { + record["bootstrap"]["operation_id"] = uuid::Uuid::new_v4().to_string().into(); + } + "incomplete" => record["complete"] = false.into(), + "pid" => record["pid"] = 123.into(), + "newer-operation" => record["intent"]["operation_id"] = uuid::Uuid::new_v4().to_string().into(), + _ => unreachable!(), + } + let bytes = serde_json::to_vec(&record).unwrap(); + std::fs::write(test.state(), &bytes).unwrap(); + let result = tokio::time::timeout( + Duration::from_millis(200), + test.worker.execute_for_tests(test.intent.clone()), + ) + .await; + assert!( + !matches!(result, Ok(Ok(_))), + "invalid {fault} receipt acknowledged without authority" + ); + assert_eq!(std::fs::read(test.state()).unwrap(), bytes); + } +} + +#[tokio::test] +async fn a_completed_receipt_cannot_stop_a_live_replacement_without_authority() { + let mut test = TestStoppedKv::new().await; + let pid = test.worker.start_kv_child_for_tests().await.unwrap(); + let result = tokio::time::timeout( + Duration::from_millis(200), + test.worker.execute_for_tests(test.intent.clone()), + ) + .await; + assert!( + !matches!(result, Ok(Ok(_))), + "a live replacement requires current authorization" + ); + assert!(std::path::Path::new(&format!("/proc/{pid}")).exists()); + test.worker.stop_kv_for_tests().await.unwrap(); +} + +#[tokio::test] +async fn retired_bootstrap_does_not_report_a_completed_receipt() { + let mut test = TestStoppedKv::new().await; + let record: serde_json::Value = serde_json::from_slice(&std::fs::read(test.state()).unwrap()).unwrap(); + let operation = record["bootstrap"]["operation_id"].as_str().unwrap(); + std::fs::write( + test.directory + .path() + .join(format!("retired-bootstrap-{operation}.json")), + b"{}", + ) + .unwrap(); + let result = tokio::time::timeout( + Duration::from_millis(200), + test.worker.execute_for_tests(test.intent.clone()), + ) + .await; + assert!( + !matches!(result, Ok(Ok(_))), + "retired local authority cannot serve the receipt" + ); +} diff --git a/container/crowdb-monitor/tests/node_service_replacement_test.rs b/container/crowdb-monitor/tests/node_service_replacement_test.rs new file mode 100644 index 000000000..9b1161d9b --- /dev/null +++ b/container/crowdb-monitor/tests/node_service_replacement_test.rs @@ -0,0 +1,107 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/service_replacement.rs"] +mod fixture; +#[path = "common/node_setup.rs"] +mod node_setup; +use fixture::TestReplacement; +use serde_json::{json, Value}; + +#[tokio::test] +async fn supervised_kv_replacements_and_legacy_receipts_refresh_the_pid_without_another_restart() { + let TestReplacement { + directory: _directory, + authority, + mut worker, + intent, + original, + health, + state, + } = TestReplacement::new().await; + worker.stop_kv_for_tests().await.unwrap(); + let replacement = worker.start_kv_child_for_tests().await.unwrap(); + assert_ne!(replacement, original); + assert!(worker.pending_for_tests(&intent).unwrap()); + assert!(worker + .owned_processes_for_tests(&intent.cluster_id, intent.node_id) + .unwrap()["process_operations"]["kv"] + .is_null()); + let result = worker.execute_for_tests(intent.clone()).await.unwrap(); + assert_eq!(result["pid"], replacement); + assert!(!worker.pending_for_tests(&intent).unwrap()); + let observed = worker + .owned_processes_for_tests(&intent.cluster_id, intent.node_id) + .unwrap(); + assert_eq!(observed["processes"]["kv"], replacement); + assert_eq!(observed["process_operations"]["kv"], intent.operation_id); + let mut record: Value = serde_json::from_slice(&std::fs::read(&state).unwrap()).unwrap(); + record.as_object_mut().unwrap().remove("bootstrap"); + std::fs::write(&state, serde_json::to_vec(&record).unwrap()).unwrap(); + assert!(worker.pending_for_tests(&intent).unwrap()); + assert_eq!( + worker.execute_for_tests(intent.clone()).await.unwrap()["pid"], + replacement + ); + let migrated: Value = serde_json::from_slice(&std::fs::read(&state).unwrap()).unwrap(); + assert_eq!(migrated["bootstrap"], json!(authority.binding.bootstrap)); + let mut foreign = migrated.clone(); + foreign["bootstrap"]["operation_id"] = json!("foreign-bootstrap"); + std::fs::write(&state, serde_json::to_vec(&foreign).unwrap()).unwrap(); + assert!(worker.pending_for_tests(&intent).unwrap()); + assert!(worker + .execute_for_tests(intent.clone()) + .await + .unwrap_err() + .to_string() + .contains("another bootstrap")); + assert_eq!( + worker + .owned_processes_for_tests(&intent.cluster_id, intent.node_id) + .unwrap()["processes"]["kv"], + replacement + ); + assert_eq!( + serde_json::from_slice::(&std::fs::read(&state).unwrap()).unwrap(), + foreign + ); + worker.stop_kv_for_tests().await.unwrap(); + health.abort(); +} + +#[tokio::test] +async fn superseded_recovery_observation_cannot_stop_the_current_process() { + let mut fixture = TestReplacement::new().await; + let mut current = fixture.intent.clone(); + current.operation_id = uuid::Uuid::new_v4().to_string(); + current.action = crowdb_protocol::mgmt::node::NodeServiceAction::Stop; + fixture + .authority + .kv + .put( + 0, + 0, + b"/deployment/services/2/kv-2", + &serde_json::to_vec(¤t).unwrap(), + None, + ) + .await + .unwrap(); + let before = std::fs::read(&fixture.state).unwrap(); + let error = fixture + .worker + .stop_unhealthy_for_tests(&fixture.intent) + .await + .unwrap_err(); + assert!(error.to_string().contains("superseded"), "{error}"); + assert_eq!( + fixture + .worker + .owned_processes_for_tests(¤t.cluster_id, current.node_id) + .unwrap()["processes"]["kv"], + fixture.original + ); + assert_eq!(std::fs::read(&fixture.state).unwrap(), before); + fixture.worker.stop_kv_for_tests().await.unwrap(); + fixture.health.abort(); +} diff --git a/container/crowdb-monitor/tests/node_setup_test.rs b/container/crowdb-monitor/tests/node_setup_test.rs index 13f98e2e3..b6dcc339b 100644 --- a/container/crowdb-monitor/tests/node_setup_test.rs +++ b/container/crowdb-monitor/tests/node_setup_test.rs @@ -175,12 +175,19 @@ async fn unbound_setup_survives_a_worker_restart_without_creating_disks_or_servi #[tokio::test] async fn diskless_setup_requires_the_exact_confirmed_admission_and_does_not_start_services() { - let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-setup-diskless"); + for kv_enabled in [false, true] { + assert_diskless_setup(kv_enabled).await; + } +} + +async fn assert_diskless_setup(kv_enabled: bool) { + let directory = run_root::TestRunRoot::new("node-setup-diskless"); let root = directory.path(); let mut choices = crowdb_protocol::mgmt::node::deployment::NodeDeploymentPolicy::default().setup; choices.storage = SetupStorage::Diskless; for service in &mut choices.services { - service.enabled = false; + service.enabled = + kv_enabled && service.kind == crowdb_protocol::mgmt::node::setup::SetupServiceKind::PaxosKv; } let inputs = crowdb_protocol::mgmt::node::deployment::NodeDeploymentOverrides { storage: Some(choices.storage.clone()), @@ -202,12 +209,17 @@ async fn diskless_setup_requires_the_exact_confirmed_admission_and_does_not_star let mut authority = node_setup::TestAuthority::start(root, &intent).await; let preparation_key = crowdb_console_shared::deployment::preparation::DEFAULT_PREPARATION_KEY; authority.kv.delete(0, 0, preparation_key, None).await.unwrap(); - let profile = profile(root); + let mut profile = profile(root); + profile.paths.run_root = root.join("run"); + let observe = kv_enabled.then(|| node_setup_control::processes_with_kv_once(&profile.paths.run_root)); NodeSetupForTests::advance_for_tests(profile.clone()) .await .unwrap(); let status = NodeSetupForTests::status_for_tests(root).unwrap().unwrap(); assert_eq!(status.phase, NodeSetupPhase::Ready); + if let Some(observe) = observe { + observe.await.unwrap(); + } assert!(matches!( authority .kv diff --git a/container/crowdb-monitor/tests/process_test.rs b/container/crowdb-monitor/tests/process_test.rs index e871a4e59..cfa45d317 100644 --- a/container/crowdb-monitor/tests/process_test.rs +++ b/container/crowdb-monitor/tests/process_test.rs @@ -204,3 +204,142 @@ async fn child_restarts_prune_old_pid_logs_without_crossing_log_channels() { assert!(directory.join("operator-notes.txt").exists()); assert_eq!(fs::read_to_string(active).unwrap(), "new"); } + +fn exit_evidence(logs: &TestLogs, pid: u32) -> serde_json::Value { + let body = fs::read_to_string(logs.0.join("monitor/monitor.log")).unwrap(); + let rows = body + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .filter(|row| row["kind"] == "child_exit_observed" && row["pid"] == pid) + .collect::>(); + assert_eq!(rows.len(), 1); + assert_eq!( + rows[0]["level"], + if rows[0]["process_exit"]["success"] == true { + "info" + } else { + "warn" + } + ); + assert_eq!(rows[0]["process_exit"]["original_pid"], pid); + rows[0]["process_exit"].clone() +} + +async fn wait_finished(manager: &mut ProcessManager) { + tokio::time::timeout(Duration::from_secs(2), async { + while manager.exit_status("fake").unwrap().is_none() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); +} + +async fn wait_marker(logs: &TestLogs) { + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if fs::read_to_string(logs.0.join("fake/service.log")).is_ok_and(|text| text.contains("ready")) { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn completed_child_evidence_preserves_original_pid_and_real_exit_codes() { + for code in [0, 7] { + let logs = TestLogs::new(); + let mut manager = ProcessManager::new(logs.0.clone(), policy(2)).await.unwrap(); + let pid = manager + .start(&service(&format!("exit {code}")), &BTreeMap::new()) + .await + .unwrap(); + wait_finished(&mut manager).await; + assert_eq!(manager.pid("fake"), None); + manager.stop("fake", Duration::from_secs(2)).await.unwrap(); + let row = exit_evidence(&logs, pid); + assert_eq!(row["status_observed"], true); + assert_eq!(row["exit_code"], code); + assert_eq!(row["success"], code == 0); + assert!(row["exit_signal"].is_null()); + assert!(row["term_sent_at_unix_ms"].is_null()); + assert!(row["kill_requested_at_unix_ms"].is_null()); + assert_eq!(row["stop_path"], "already_exited_before_stop"); + } +} + +#[tokio::test] +async fn owned_term_stop_records_the_real_signal_without_success() { + let logs = TestLogs::new(); + let mut manager = ProcessManager::new(logs.0.clone(), policy(2)).await.unwrap(); + let pid = manager + .start(&service("printf ready; exec sleep 30"), &BTreeMap::new()) + .await + .unwrap(); + wait_marker(&logs).await; + manager.stop("fake", Duration::from_secs(2)).await.unwrap(); + let row = exit_evidence(&logs, pid); + assert_eq!(row["exit_signal"], 15); + assert_eq!(row["success"], false); + assert!(row["exit_code"].is_null()); + assert!(row["term_sent_at_unix_ms"].is_number()); + assert!(row["kill_requested_at_unix_ms"].is_null()); + assert_eq!(row["stop_path"], "owned_stop_requested"); + assert_eq!(row["grace_ms"], 2000); +} + +#[tokio::test] +async fn ignored_term_retains_original_grace_and_reports_kill_escalation() { + let logs = TestLogs::new(); + let mut manager = ProcessManager::new(logs.0.clone(), policy(2)).await.unwrap(); + let pid = manager + .start( + &service("trap '' TERM; printf ready; exec sleep 30"), + &BTreeMap::new(), + ) + .await + .unwrap(); + wait_marker(&logs).await; + manager.stop("fake", Duration::from_millis(30)).await.unwrap(); + let row = exit_evidence(&logs, pid); + assert_eq!(row["exit_signal"], 9); + assert_eq!(row["success"], false); + assert!(row["exit_code"].is_null()); + assert!(row["term_sent_at_unix_ms"].is_number()); + assert!(row["kill_requested_at_unix_ms"].is_number()); + assert_eq!(row["stop_path"], "owned_stop_grace_elapsed"); + assert_eq!(row["grace_ms"], 30); + assert!(row["elapsed_since_term_ms"].as_u64().unwrap() >= 30); +} + +#[tokio::test] +async fn self_signaled_exit_is_not_mislabeled_as_supervisor_kill() { + let logs = TestLogs::new(); + let mut manager = ProcessManager::new(logs.0.clone(), policy(2)).await.unwrap(); + let pid = manager + .start(&service("kill -KILL $$"), &BTreeMap::new()) + .await + .unwrap(); + wait_finished(&mut manager).await; + manager.stop("fake", Duration::from_secs(2)).await.unwrap(); + let row = exit_evidence(&logs, pid); + assert_eq!(row["exit_signal"], 9); + assert_eq!(row["success"], false); + assert!(row["exit_code"].is_null()); + assert!(row["term_sent_at_unix_ms"].is_null()); + assert!(row["kill_requested_at_unix_ms"].is_null()); + assert_eq!(row["stop_path"], "already_exited_before_stop"); +} + +#[tokio::test] +async fn unowned_stop_does_not_fabricate_exit_evidence() { + let logs = TestLogs::new(); + let mut manager = ProcessManager::new(logs.0.clone(), policy(2)).await.unwrap(); + assert!(manager.stop("missing", Duration::from_secs(2)).await.is_err()); + let body = fs::read_to_string(logs.0.join("monitor/monitor.log")).unwrap(); + assert!(!body.contains("child_exit_observed")); + assert!(!body.contains("\"success\":true")); +} diff --git a/container/oci-image/Dockerfile b/container/oci-image/Dockerfile index 5299f7806..47a2e030a 100644 --- a/container/oci-image/Dockerfile +++ b/container/oci-image/Dockerfile @@ -45,6 +45,6 @@ ENV PATH="/opt/crowdb/bin:${PATH}" \ CROWDB_RUNTIME_ROOT="/opt/crowdb/run" USER root VOLUME ["/opt/crowdb/data"] -EXPOSE 9090 9091 9092 9093 9095 2222 +EXPOSE 9090 16000 17000 18000 9095 2222 STOPSIGNAL SIGTERM ENTRYPOINT ["/usr/bin/tini", "--", "/opt/crowdb/bin/entrypoint"] diff --git a/container/oci-image/README.md b/container/oci-image/README.md index 0a4fc3ae1..9d634992d 100644 --- a/container/oci-image/README.md +++ b/container/oci-image/README.md @@ -110,6 +110,7 @@ pixi run oci-run-1c --cluster demo pixi run oci-run-3c --cluster demo-three --port-base 9190 pixi run oci-stop --cluster demo-three pixi run oci-run-3c --cluster demo-three +pixi run oci-stop --cluster demo-three pixi run oci-clean --cluster demo-three --keep-data # Recreate containers and bridge with the original volumes and accepted policy: pixi run oci-run-3c --cluster demo-three @@ -123,28 +124,39 @@ A different node count or conflicting input requires another cluster name. Ports for each node are UI/S3/Iceberg/Dataset at base/base+1/base+2/base+3; subsequent members add 10 and 20. All published ports bind to `127.0.0.1`. Run and stop default to `local`; stop accepts `--all` for every recorded local -cluster. Clean defaults to every recorded local and host-network cluster and -deletes their node data; use `--cluster NAME` to clean only one. Explicit `--all` -also selects every recorded cluster. Use `--keep-data` to preserve data and -restart receipts; `--delete-data` explicitly selects the default deletion mode. +cluster. Default clean skips running recorded local and host-network clusters, +including all resources of a bridge cluster with any running member. It removes +stopped clusters and their node data, then cleans unused resources on the selected +Docker daemon, including containers without CROWDB receipts: ```bash pixi run oci-clean ``` -With no cluster receipts, clean reports that there are no recorded clusters and -checks for dangling CROWDB images if Docker is installed. An explicitly named -cluster without a receipt remains an error. Containerd-only host cleanup needs -no Docker daemon. +- Remove stopped containers first, then unused networks and both named and + anonymous volumes, followed by unused Docker build cache. Running containers + and referenced resources remain intact. Docker removals never force active + containers or in-use resources; a concurrent new reference makes removal fail. +- Preserve resources belonging to skipped running clusters and their restart + receipts, including detached volumes and stopped members of those clusters. +- Use `--keep-data` to preserve all Docker volumes, node data and restart receipts; + `--delete-data` explicitly selects the default deletion mode. +- `--cluster NAME` explicitly stops and removes one recorded cluster, without + daemon-wide garbage collection. `--all` explicitly stops and removes every + recorded cluster and also cleans unused Docker resources. +- With no cluster receipts, unused Docker resource cleanup still runs if Docker + is installed. An explicitly named cluster without a receipt remains an error. + Containerd-only host cleanup needs no Docker daemon. Host receipts live in `.crowdb-runtime/persistent/oci-clusters/`. The shortcuts verify the Docker daemon/context and exact ownership labels before effects. -Clean stops and removes owned containers, unused owned bridges and owned data +Clean removes selected stopped containers, unused owned bridges and owned data volumes. It also removes the launch receipt and matching optional CLI target binding. `--keep-data` retains volumes and restart inputs. Image cleanup only considers dangling CROWDB images, preserving retained cluster references and images used by any running or stopped container. Attached or relabelled -networks and volumes referenced elsewhere are rejected. No global prune runs. +networks and volumes referenced elsewhere are rejected. Daemon-wide garbage +collection runs only when no `--cluster` is selected. These host scripts invoke Docker directly and require no CLI binary or application target binding. A binding with different ownership is rejected before data deletion. @@ -262,8 +274,11 @@ A direct single-container cluster needs a volume and four published ports: ```bash pixi run docker run -d --name crowdb-one --cpus 1 --memory 1536m \ - -p 127.0.0.1:9090:9090 -p 127.0.0.1:9091:9091 \ - -p 127.0.0.1:9092:9092 -p 127.0.0.1:9093:9093 \ + -e CROWDB_S3_PUBLIC_URI=http://127.0.0.1:9091 \ + -e CROWDB_ICEBERG_PUBLIC_URI=http://127.0.0.1:9092 \ + -e CROWDB_DATASET_PUBLIC_URI=http://127.0.0.1:9093 \ + -p 127.0.0.1:9090:9090 -p 127.0.0.1:9091:16000 \ + -p 127.0.0.1:9092:17000 -p 127.0.0.1:9093:18000 \ --mount type=volume,source=crowdb-one-data,target=/opt/crowdb/data \ crowdb-node:dev ``` @@ -278,24 +293,24 @@ pixi run docker run -d --name crowdb-demo-1 --network crowdb-demo --cpus 1 --mem -e CROWDB_S3_PUBLIC_URI=http://127.0.0.1:9091 \ -e CROWDB_ICEBERG_PUBLIC_URI=http://127.0.0.1:9092 \ -e CROWDB_DATASET_PUBLIC_URI=http://127.0.0.1:9093 \ - -p 127.0.0.1:9090:9090 -p 127.0.0.1:9091:9091 \ - -p 127.0.0.1:9092:9092 -p 127.0.0.1:9093:9093 \ + -p 127.0.0.1:9090:9090 -p 127.0.0.1:9091:16000 \ + -p 127.0.0.1:9092:17000 -p 127.0.0.1:9093:18000 \ --mount type=volume,source=crowdb-demo-1-data,target=/opt/crowdb/data crowdb-node:dev pixi run docker run -d --name crowdb-demo-2 --network crowdb-demo --cpus 1 --memory 1536m \ -e CROWDB_CLUSTER=demo -e CROWDB_INITIAL_VOTERS=3 \ -e CROWDB_S3_PUBLIC_URI=http://127.0.0.1:9101 \ -e CROWDB_ICEBERG_PUBLIC_URI=http://127.0.0.1:9102 \ -e CROWDB_DATASET_PUBLIC_URI=http://127.0.0.1:9103 \ - -p 127.0.0.1:9100:9090 -p 127.0.0.1:9101:9091 \ - -p 127.0.0.1:9102:9092 -p 127.0.0.1:9103:9093 \ + -p 127.0.0.1:9100:9090 -p 127.0.0.1:9101:16000 \ + -p 127.0.0.1:9102:17000 -p 127.0.0.1:9103:18000 \ --mount type=volume,source=crowdb-demo-2-data,target=/opt/crowdb/data crowdb-node:dev pixi run docker run -d --name crowdb-demo-3 --network crowdb-demo --cpus 1 --memory 1536m \ -e CROWDB_CLUSTER=demo -e CROWDB_INITIAL_VOTERS=3 \ -e CROWDB_S3_PUBLIC_URI=http://127.0.0.1:9111 \ -e CROWDB_ICEBERG_PUBLIC_URI=http://127.0.0.1:9112 \ -e CROWDB_DATASET_PUBLIC_URI=http://127.0.0.1:9113 \ - -p 127.0.0.1:9110:9090 -p 127.0.0.1:9111:9091 \ - -p 127.0.0.1:9112:9092 -p 127.0.0.1:9113:9093 \ + -p 127.0.0.1:9110:9090 -p 127.0.0.1:9111:16000 \ + -p 127.0.0.1:9112:17000 -p 127.0.0.1:9113:18000 \ --mount type=volume,source=crowdb-demo-3-data,target=/opt/crowdb/data crowdb-node:dev ``` @@ -309,6 +324,16 @@ pixi run docker compose -f container/oci-image/compose-three.yaml up -d pixi run docker compose -f container/oci-image/compose-three.yaml down ``` +Fresh nodes listen on container ports 9090 (Web), 16000 (S3), 17000 (Iceberg) +and 18000 (Dataset). The examples retain host ports 9090–9093. New local Docker +launch receipts use version 2; existing version-1 receipts retain their original +container ports and pinned image when resumed or recreated. + +For Compose volumes with accepted legacy listeners, set +`CROWDB_S3_CONTAINER_PORT=9091`, `CROWDB_ICEBERG_CONTAINER_PORT=9092` and +`CROWDB_DATASET_CONTAINER_PORT=9093` to match that policy. Public-origin variables +still describe the host-side ports. Retained policy is not rewritten on restart. + `down` removes the Compose containers and network, preserving named volumes. Set `CROWDB_IMAGE` to select the loaded image and `CROWDB_CLUSTER` to isolate another project. Change the port variables shown in each file before starting diff --git a/container/oci-image/clean.py b/container/oci-image/clean.py index 9a313d287..3eef66c41 100644 --- a/container/oci-image/clean.py +++ b/container/oci-image/clean.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Copyright 2026-present Gian # Licensed under the Apache License, Version 2.0. -"""Clean recorded Docker and containerd clusters and their data by default.""" +"""Clean stopped clusters and unused Docker resources; preserve running clusters.""" import argparse from pathlib import Path @@ -9,6 +9,7 @@ import sys from docker_local_cleanup import clean, image_garbage, selected +from docker_garbage import collect from docker_local_state import Docker, RECORDS, check_daemon, load from host_cleanup import check_data_users, clean as clean_host, selected_records from host_data_cleanup import HostDataCleanup @@ -18,13 +19,16 @@ def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) scope = parser.add_mutually_exclusive_group() - scope.add_argument('--cluster', help='clean only this recorded cluster') - scope.add_argument('--all', action='store_true', help='clean every recorded cluster (default)') + scope.add_argument('--cluster', help='stop and clean only this recorded cluster') + scope.add_argument('--all', action='store_true', + help='stop and clean every recorded cluster, then clean unused Docker resources') data = parser.add_mutually_exclusive_group() data.add_argument('--keep-data', action='store_true', help='preserve data and restart receipts') - data.add_argument('--delete-data', action='store_true', help='delete owned data (default)') + data.add_argument('--delete-data', action='store_true', + help='delete stopped-cluster data and unused Docker volumes (default)') parser.add_argument('--runtime-root', type=Path, default=RECORDS.parents[1]) args = parser.parse_args(argv) + stop_running = args.cluster is not None or args.all directory = args.runtime_root / 'persistent/oci-clusters' if args.cluster is None: records = selected(None, True, directory) @@ -40,20 +44,39 @@ def main(argv=None): if docker is not None: for record in records: check_daemon(docker, record) + runtime_identity = docker.identity() if docker is not None else None + retained = [] + if not stop_running and docker is not None: + stopped = [] + for record in records: + if any(item['State']['Running'] for item in docker.owned(record, 'container')): + print(f"Skipped running cluster {record['cluster']}") + retained.append(record) + else: + stopped.append(record) + records = stopped plans = [] + stopped_hosts = [] + retained_hosts = [] for record in hosts: runtime = HostRuntime(record['options'], record['cluster']) runtime.check(record) current = runtime.named(record['name']) if current is not None: require_owned(current, record) + if not stop_running and current['State']['Running']: + print(f"Skipped running host cluster {record['cluster']}") + retained_hosts.append(record) + continue if not args.keep_data: check_data_users(runtime, record) plans.append(None if args.keep_data else HostDataCleanup(record, args.runtime_root)) + stopped_hosts.append(record) + hosts = stopped_hosts for record in records: - clean(docker, record, not args.keep_data, directory) + clean(docker, record, not args.keep_data, directory, stop_running=stop_running) for record, plan in zip(hosts, plans): - clean_host(record) + clean_host(record, stop_running=stop_running) if plan is not None: runtime = HostRuntime(record['options'], record['cluster']) runtime.check(record) @@ -62,10 +85,14 @@ def main(argv=None): print(f"Cleaned host container for {record['cluster']}; " + ('preserved data and restart receipts' if args.keep_data else 'deleted node data and receipts; preserved supplied disk files and devices')) - if not records and not hosts: + if not records and not hosts and not retained and not retained_hosts: print('No recorded CROWDB clusters to clean') if docker is not None: - image_garbage(docker, directory, runtime=records[0]['runtime'] if records else docker.identity()) + if args.cluster is None: + collect(docker, retained + (records if args.keep_data else []), + retained_hosts + (hosts if args.keep_data else []), + delete_data=not args.keep_data, runtime=runtime_identity) + image_garbage(docker, directory, runtime=runtime_identity) else: print('No local Docker runtime selected; skipped dangling Docker image cleanup') diff --git a/container/oci-image/compose-three.yaml b/container/oci-image/compose-three.yaml index af59582c7..01af56fff 100644 --- a/container/oci-image/compose-three.yaml +++ b/container/oci-image/compose-three.yaml @@ -17,9 +17,9 @@ services: CROWDB_DATASET_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE1_DATASET_PORT:-9093} ports: - "127.0.0.1:${CROWDB_NODE1_UI_PORT:-9090}:9090" - - "127.0.0.1:${CROWDB_NODE1_S3_PORT:-9091}:9091" - - "127.0.0.1:${CROWDB_NODE1_ICEBERG_PORT:-9092}:9092" - - "127.0.0.1:${CROWDB_NODE1_DATASET_PORT:-9093}:9093" + - "127.0.0.1:${CROWDB_NODE1_S3_PORT:-9091}:${CROWDB_S3_CONTAINER_PORT:-16000}" + - "127.0.0.1:${CROWDB_NODE1_ICEBERG_PORT:-9092}:${CROWDB_ICEBERG_CONTAINER_PORT:-17000}" + - "127.0.0.1:${CROWDB_NODE1_DATASET_PORT:-9093}:${CROWDB_DATASET_CONTAINER_PORT:-18000}" volumes: - node1-data:/opt/crowdb/data node2: @@ -32,9 +32,9 @@ services: CROWDB_DATASET_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE2_DATASET_PORT:-9103} ports: - "127.0.0.1:${CROWDB_NODE2_UI_PORT:-9100}:9090" - - "127.0.0.1:${CROWDB_NODE2_S3_PORT:-9101}:9091" - - "127.0.0.1:${CROWDB_NODE2_ICEBERG_PORT:-9102}:9092" - - "127.0.0.1:${CROWDB_NODE2_DATASET_PORT:-9103}:9093" + - "127.0.0.1:${CROWDB_NODE2_S3_PORT:-9101}:${CROWDB_S3_CONTAINER_PORT:-16000}" + - "127.0.0.1:${CROWDB_NODE2_ICEBERG_PORT:-9102}:${CROWDB_ICEBERG_CONTAINER_PORT:-17000}" + - "127.0.0.1:${CROWDB_NODE2_DATASET_PORT:-9103}:${CROWDB_DATASET_CONTAINER_PORT:-18000}" volumes: - node2-data:/opt/crowdb/data node3: @@ -47,9 +47,9 @@ services: CROWDB_DATASET_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE3_DATASET_PORT:-9113} ports: - "127.0.0.1:${CROWDB_NODE3_UI_PORT:-9110}:9090" - - "127.0.0.1:${CROWDB_NODE3_S3_PORT:-9111}:9091" - - "127.0.0.1:${CROWDB_NODE3_ICEBERG_PORT:-9112}:9092" - - "127.0.0.1:${CROWDB_NODE3_DATASET_PORT:-9113}:9093" + - "127.0.0.1:${CROWDB_NODE3_S3_PORT:-9111}:${CROWDB_S3_CONTAINER_PORT:-16000}" + - "127.0.0.1:${CROWDB_NODE3_ICEBERG_PORT:-9112}:${CROWDB_ICEBERG_CONTAINER_PORT:-17000}" + - "127.0.0.1:${CROWDB_NODE3_DATASET_PORT:-9113}:${CROWDB_DATASET_CONTAINER_PORT:-18000}" volumes: - node3-data:/opt/crowdb/data volumes: diff --git a/container/oci-image/compose.yaml b/container/oci-image/compose.yaml index 7ad711291..96e5e608f 100644 --- a/container/oci-image/compose.yaml +++ b/container/oci-image/compose.yaml @@ -15,9 +15,9 @@ services: CROWDB_DATASET_PUBLIC_URI: http://127.0.0.1:${CROWDB_DATASET_PORT:-9093} ports: - "127.0.0.1:${CROWDB_UI_PORT:-9090}:9090" - - "127.0.0.1:${CROWDB_S3_PORT:-9091}:9091" - - "127.0.0.1:${CROWDB_ICEBERG_PORT:-9092}:9092" - - "127.0.0.1:${CROWDB_DATASET_PORT:-9093}:9093" + - "127.0.0.1:${CROWDB_S3_PORT:-9091}:${CROWDB_S3_CONTAINER_PORT:-16000}" + - "127.0.0.1:${CROWDB_ICEBERG_PORT:-9092}:${CROWDB_ICEBERG_CONTAINER_PORT:-17000}" + - "127.0.0.1:${CROWDB_DATASET_PORT:-9093}:${CROWDB_DATASET_CONTAINER_PORT:-18000}" volumes: - node-data:/opt/crowdb/data volumes: diff --git a/container/oci-image/docker_garbage.py b/container/oci-image/docker_garbage.py new file mode 100644 index 000000000..9206f76a1 --- /dev/null +++ b/container/oci-image/docker_garbage.py @@ -0,0 +1,63 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Reclaim unused Docker resources without stopping containers or retained clusters.""" + +import json + +from docker_local_state import OWNER, labels, resource_names +from runtime import OWNER as HOST_OWNER + + +def collect(docker, records, hosts, *, delete_data=True, runtime): + """Keep retained receipts and let non-force Docker removals guard reference races.""" + owners = {record['owner'] for record in (*records, *hosts)} + networks = {name for record in records for name in resource_names(record, 'network')} + volumes = {name for record in records for name in resource_names(record, 'volume')} + + def retained(item, kind): + metadata = labels(item, kind) + return any(metadata.get(key) in owners for key in (OWNER, HOST_OWNER)) + + def check_runtime(): + if docker.identity() != runtime: + raise ValueError('Docker endpoint/context changed during garbage cleanup') + + def remove(kind, identity): + check_runtime() + # Never force removal: Docker rejects containers that restarted and + # volumes/networks that acquired a reference since inspection. + print(f'Removing unused Docker {kind}: {identity}', flush=True) + result = docker.call('rm', identity) if kind == 'container' else docker.call(kind, 'rm', identity) + if result: + print(result, flush=True) + + check_runtime() + for candidate in docker.objects('container'): + if retained(candidate, 'container'): + continue + item = json.loads(docker.call('container', 'inspect', candidate['Id']))[0] + if item['State'].get('Status') not in ('created', 'exited', 'dead'): + continue + if item['State']['Running'] or retained(item, 'container'): + continue + remove('container', item['Id']) + + for kind, protected in (('network', networks), ('volume', volumes)): + if kind == 'volume' and not delete_data: + print('Preserved unused Docker volumes (--keep-data)', flush=True) + continue + for candidate in docker.objects(kind, ('dangling=true',)): + identity = candidate['Name'] if kind == 'volume' else candidate['Id'] + item = json.loads(docker.call(kind, 'inspect', identity))[0] + if item['Name'] in protected or retained(item, kind): + continue + if kind == 'network': + if item.get('Containers') or item['Name'] in ('bridge', 'host', 'none'): + continue + elif docker.call('ps', '-aq', '--filter', 'volume=' + item['Name']): + continue + remove(kind, identity) + + check_runtime() + print('Pruning unused Docker build cache', flush=True) + print(docker.call('builder', 'prune', '--all', '--force'), flush=True) diff --git a/container/oci-image/docker_local_cleanup.py b/container/oci-image/docker_local_cleanup.py index c86d78078..d3f5706b4 100644 --- a/container/oci-image/docker_local_cleanup.py +++ b/container/oci-image/docker_local_cleanup.py @@ -36,9 +36,13 @@ def stop(docker, record): docker.call("stop", "--time", "20", item["Id"]) -def clean(docker, record, delete_data=False, directory=RECORDS): +def clean(docker, record, delete_data=False, directory=RECORDS, *, stop_running=True): binding = owned_binding(record, directory) if delete_data else None - stop(docker, record) + check_daemon(docker, record) + if stop_running: + stop(docker, record) + elif any(item['State']['Running'] for item in docker.owned(record, 'container')): + raise ValueError('cluster is running; refusing cleanup') for item in docker.owned(record, "container"): item = current(docker, record, "container", item["Id"]) if item["State"]["Running"]: diff --git a/container/oci-image/docker_local_launch.py b/container/oci-image/docker_local_launch.py index a5242464f..99222ace3 100644 --- a/container/oci-image/docker_local_launch.py +++ b/container/oci-image/docker_local_launch.py @@ -8,6 +8,12 @@ from docker_local_state import CLUSTER, OWNER, check_daemon, load, require_owned, seal +CONTAINER_PORTS = (9090, 16000, 17000, 18000) + + +def container_ports(record): + return (9090, 9091, 9092, 9093) if record["version"] == 1 else CONTAINER_PORTS + def arguments(record): return ["--label", f"{CLUSTER}={record['cluster']}", "--label", f"{OWNER}={record['owner']}"] @@ -43,7 +49,7 @@ def accepted(docker, args): raise ValueError("local node image must be Linux amd64") if metadata.get("Config", {}).get("Labels", {}).get("org.opencontainers.image.source") != "https://github.com/buzzcrow/crowdb": raise ValueError("image is not a CROWDB node image") - record = {"version": 1, "cluster": args.cluster, "owner": str(uuid.uuid4()), "runtime": docker.identity(), + record = {"version": 2, "cluster": args.cluster, "owner": str(uuid.uuid4()), "runtime": docker.identity(), "nodes": args.nodes if args.nodes is not None else 1, "port_base": args.port_base if args.port_base is not None else 9090, "image": image, "image_id": metadata["Id"], "cpus": args.cpus if args.cpus is not None else 1, "memory_mib": args.memory_mib if args.memory_mib is not None else 1536} @@ -59,8 +65,8 @@ def verify_container(item, record, index, volume, network): require_owned(item, record, "container") host = item["HostConfig"] base = record["port_base"] + (index - 1) * 10 - ports = {f"{9090+offset}/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(base+offset)}] - for offset in range(4)} + ports = {f"{port}/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(base+offset)}] + for offset, port in enumerate(container_ports(record))} expected_environment = {"CROWDB_CLUSTER": record["cluster"], "CROWDB_INITIAL_VOTERS": str(record["nodes"])} for offset, protocol in enumerate((None, "S3", "ICEBERG", "DATASET")): if protocol: @@ -102,7 +108,7 @@ def run(docker, record): "--mount", f"type=volume,source={volume},target=/opt/crowdb/data", "-e", "CROWDB_CLUSTER=" + record["cluster"], "-e", f"CROWDB_INITIAL_VOTERS={record['nodes']}"] for offset, protocol in enumerate((None, "S3", "ICEBERG", "DATASET")): - command += ["-p", f"127.0.0.1:{base+offset}:{9090+offset}"] + command += ["-p", f"127.0.0.1:{base+offset}:{container_ports(record)[offset]}"] if protocol: command += ["-e", f"CROWDB_{protocol}_PUBLIC_URI=http://127.0.0.1:{base+offset}"] check_daemon(docker, record) diff --git a/container/oci-image/docker_local_state.py b/container/oci-image/docker_local_state.py index 0605f0357..960492c6f 100644 --- a/container/oci-image/docker_local_state.py +++ b/container/oci-image/docker_local_state.py @@ -71,7 +71,7 @@ def load(cluster, directory=RECORDS): if not stat.S_ISREG(metadata.st_mode) or metadata.st_mode & 0o077 or metadata.st_size > 1024 * 1024: raise ValueError("cluster receipt must be a bounded private regular file") record = json.loads(path.read_text()) - if record.get("version") != 1 or record.get("cluster") != cluster: + if record.get("version") not in (1, 2) or record.get("cluster") != cluster: raise ValueError("invalid cluster receipt") required = {"version", "cluster", "owner", "runtime", "nodes", "port_base", "image", "image_id", "cpus", "memory_mib"} if set(record) != required or not isinstance(record["runtime"], dict): diff --git a/container/oci-image/host_cleanup.py b/container/oci-image/host_cleanup.py index 88c1b4a34..ef95de4b8 100644 --- a/container/oci-image/host_cleanup.py +++ b/container/oci-image/host_cleanup.py @@ -68,7 +68,7 @@ def selected_records(cluster, root): return [selected(path.stem, root) for path in sorted(directory.glob('*.json'))] -def clean(record): +def clean(record, *, stop_running=True): runtime = HostRuntime(record['options'], record['cluster']) runtime.check(record) current = runtime.named(record['name']) @@ -77,6 +77,8 @@ def clean(record): require_owned(current, record) identity = current['Id'] if current['State']['Running']: + if not stop_running: + raise ValueError('host container is running; refusing cleanup') runtime.check(record) runtime.invoke('stop', identity) current = runtime.inspect(identity) diff --git a/container/oci-image/run-node.py b/container/oci-image/run-node.py index 0c408143d..fe5a94d39 100755 --- a/container/oci-image/run-node.py +++ b/container/oci-image/run-node.py @@ -20,9 +20,13 @@ def digest_reference(reference): return digest.startswith("sha256:") and len(digest) == 71 and all(c in "0123456789abcdef" for c in digest[7:]) -# Fixed in-image listeners; host networking admits only one node per host. -HOST_PORTS = (2222, 7100, 7101, 7200, 7300, 7301, 7400, 7401, - 9090, 9091, 9092, 9093, 9094, 9095, 10000, 10100) +# Base listeners used by one node. Host-network instances add +# ``CROWDB_PORT_OFFSET`` to every listener before the policy is sealed. These +# values mirror the canonical service blocks in +# ``lib/crowdb-protocol/src/port/ports.rs``. +HOST_PORTS = (2222, 9090, 9095, 10000, 10100, 10101, 11000, 11100, 11200, + 12000, 12100, 12200, 13000, 14000, 15100, 15200, 16000, 17000, + 18000, 19000) def check_host_ports(ports=HOST_PORTS): @@ -37,6 +41,15 @@ def check_host_ports(ports=HOST_PORTS): raise ValueError(f"host node TCP port {port} is already occupied") from error +def shifted_ports(offset): + if not isinstance(offset, int) or offset < 0: + raise ValueError("port offset must be a non-negative integer") + ports = tuple(port + offset for port in HOST_PORTS) + if any(port > 65535 for port in ports): + raise ValueError("port offset shifts a node listener beyond port 65535") + return ports + + def validate(args, parser, check_ports=True): if (platform.system(), platform.machine()) != ("Linux", "x86_64"): parser.error("Linux amd64 host required; macOS VM backend is reserved") @@ -53,6 +66,10 @@ def validate(args, parser, check_ports=True): parser.error("CPU shares must be between 2 and 262144") if args.worker_threads is not None and args.worker_threads < 1: parser.error("worker thread count must be positive") + try: + ports = shifted_ports(getattr(args, "port_offset", 0)) + except ValueError as error: + parser.error(str(error)) available = os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE") // (1024 * 1024) if not (0 < args.cpus <= (os.cpu_count() or 1)) or not 512 <= args.memory_mib <= available: parser.error("CPU and memory boundaries must fit the host; memory must be at least 512 MiB") @@ -83,7 +100,7 @@ def validate(args, parser, check_ports=True): try: socket.if_nametoindex(args.interface) if check_ports: - check_host_ports() + check_host_ports(ports) except (OSError, ValueError) as error: parser.error(str(error)) elif args.runtime != "docker": @@ -118,6 +135,8 @@ def main(): parser.add_argument("--seed", action="append", default=[]) parser.add_argument("--cluster", required=True) parser.add_argument("--initial-voters", type=int, choices=(1, 3), default=3) + parser.add_argument("--port-offset", type=int, default=0, + help="Add this offset to every host-network node listener") args = parser.parse_args() validate(args, parser) runtime = Runtime(args.runtime, args.address, "crowdb-" + args.cluster, args.snapshotter) diff --git a/container/oci-image/runtime.py b/container/oci-image/runtime.py index fe6b8b241..db627149d 100644 --- a/container/oci-image/runtime.py +++ b/container/oci-image/runtime.py @@ -119,7 +119,8 @@ def create(self, args): "-e", f"CROWDB_INITIAL_VOTERS={args.initial_voters}", "-e", "CROWDB_DEPLOYMENT_MODE=production", "-e", f"CROWDB_PHYSICAL_HOST_ID={args.physical_host_id}", - "-e", f"CROWDB_MANAGEMENT_INTERFACE={args.interface}"] + "-e", f"CROWDB_MANAGEMENT_INTERFACE={args.interface}", + "-e", f"CROWDB_PORT_OFFSET={getattr(args, 'port_offset', 0)}"] workers = getattr(args, "worker_threads", None) if workers is not None: command += ["-e", f"TOKIO_WORKER_THREADS={workers}"] diff --git a/container/oci-image/tests/cleanup_cli_test.py b/container/oci-image/tests/cleanup_cli_test.py index 9b2eeaf32..267bb2a77 100644 --- a/container/oci-image/tests/cleanup_cli_test.py +++ b/container/oci-image/tests/cleanup_cli_test.py @@ -1,6 +1,6 @@ # Copyright 2026-present Gian # Licensed under the Apache License, Version 2.0. -"""Clean command defaults select all receipts and remove owned data.""" +"""Default cleanup preserves running clusters and reclaims unused Docker resources.""" import contextlib import importlib.util @@ -37,20 +37,26 @@ def setUp(self): patch.object(cleanup, 'HostRuntime'), patch.object(cleanup, 'HostDataCleanup'), patch.object(cleanup, 'clean_host'), patch.object(cleanup, 'require_owned'), patch.object(cleanup.shutil, 'which', return_value='/docker'), - patch.object(cleanup, 'check_data_users')] + patch.object(cleanup, 'check_data_users'), patch.object(cleanup, 'collect')] (self.factory, self.select, self.check, self.clean, self.select_hosts, self.garbage, - self.load, self.runtime, self.data, self.clean_host, self.require, self.which, self.users) = [ + self.load, self.runtime, self.data, self.clean_host, self.require, self.which, self.users, + self.collect) = [ item.start() for item in self.patches] + self.runtime.return_value.named.return_value = None self.addCleanup(lambda: [item.stop() for item in reversed(self.patches)]) def test_bare_clean_selects_all_and_deletes_data_before_image_garbage(self): cleanup.main([]) self.select.assert_called_once_with(None, True, cleanup.RECORDS) self.assertEqual(self.clean.call_args_list, - [call(self.docker, record, True, cleanup.RECORDS) for record in self.records]) - self.clean_host.assert_called_once_with(self.hosts[0]) + [call(self.docker, record, True, cleanup.RECORDS, stop_running=False) + for record in self.records]) + self.clean_host.assert_called_once_with(self.hosts[0], stop_running=False) self.data.return_value.remove.assert_called_once_with() - self.garbage.assert_called_once_with(self.docker, cleanup.RECORDS, runtime=self.records[0]['runtime']) + self.collect.assert_called_once_with(self.docker, [], [], delete_data=True, + runtime=self.docker.identity.return_value) + self.garbage.assert_called_once_with(self.docker, cleanup.RECORDS, + runtime=self.docker.identity.return_value) def test_explicit_cluster_and_keep_data(self): self.load.return_value = self.records[0] @@ -59,14 +65,17 @@ def test_explicit_cluster_and_keep_data(self): self.select.assert_not_called() self.load.assert_called_once_with('first', cleanup.RECORDS) self.select_hosts.assert_called_once_with('first', cleanup.RECORDS.parents[1]) - self.clean.assert_called_once_with(self.docker, self.records[0], False, cleanup.RECORDS) + self.clean.assert_called_once_with(self.docker, self.records[0], False, cleanup.RECORDS, + stop_running=True) self.data.assert_not_called() + self.collect.assert_not_called() def test_explicit_all_and_delete_data(self): cleanup.main(['--all', '--delete-data']) self.select.assert_called_once_with(None, True, cleanup.RECORDS) self.assertEqual(self.clean.call_args_list, - [call(self.docker, record, True, cleanup.RECORDS) for record in self.records]) + [call(self.docker, record, True, cleanup.RECORDS, stop_running=True) + for record in self.records]) def test_empty_inventory_still_checks_image_garbage(self): self.select.return_value = [] @@ -84,13 +93,13 @@ def test_containerd_only_cleanup_needs_no_docker_daemon(self): cleanup.main([]) self.factory.assert_not_called() self.garbage.assert_not_called() - self.clean_host.assert_called_once_with(self.hosts[0]) + self.clean_host.assert_called_once_with(self.hosts[0], stop_running=False) self.data.return_value.remove.assert_called_once_with() def test_host_keep_data_preserves_restart_receipts(self): self.load.return_value = None cleanup.main(['--cluster', 'host', '--keep-data']) - self.clean_host.assert_called_once_with(self.hosts[0]) + self.clean_host.assert_called_once_with(self.hosts[0], stop_running=True) self.data.assert_not_called() def test_host_docker_endpoint_does_not_trigger_image_cleanup_on_current_docker_context(self): @@ -98,7 +107,7 @@ def test_host_docker_endpoint_does_not_trigger_image_cleanup_on_current_docker_c self.hosts[0]['options']['runtime'] = 'docker' cleanup.main([]) self.factory.assert_not_called() - self.clean_host.assert_called_once_with(self.hosts[0]) + self.clean_host.assert_called_once_with(self.hosts[0], stop_running=False) self.garbage.assert_not_called() def test_missing_explicit_cluster_is_an_error_before_runtime_access(self): @@ -109,6 +118,30 @@ def test_missing_explicit_cluster_is_an_error_before_runtime_access(self): self.factory.assert_not_called() self.runtime.assert_not_called() + def test_default_retains_running_cluster_and_all_its_resources(self): + self.docker.owned.side_effect = [[dict(State=dict(Running=True))], []] + self.select_hosts.return_value = [] + cleanup.main([]) + self.clean.assert_called_once_with(self.docker, self.records[1], True, cleanup.RECORDS, + stop_running=False) + self.collect.assert_called_once_with(self.docker, self.records[:1], [], delete_data=True, + runtime=self.docker.identity.return_value) + + def test_default_retains_running_host_data_without_planning_deletion(self): + self.runtime.return_value.named.return_value = dict(State=dict(Running=True)) + cleanup.main([]) + self.clean_host.assert_not_called() + self.data.assert_not_called() + self.users.assert_not_called() + self.collect.assert_called_once_with(self.docker, [], self.hosts, delete_data=True, + runtime=self.docker.identity.return_value) + + def test_default_keep_data_protects_receipts_from_global_collection(self): + cleanup.main(['--keep-data']) + self.data.assert_not_called() + self.collect.assert_called_once_with(self.docker, self.records, self.hosts, delete_data=False, + runtime=self.docker.identity.return_value) + def test_all_runtime_identities_are_checked_before_cleanup(self): self.check.side_effect = [None, ValueError('changed daemon')] with self.assertRaisesRegex(ValueError, 'changed daemon'): diff --git a/container/oci-image/tests/docker_garbage_test.py b/container/oci-image/tests/docker_garbage_test.py new file mode 100644 index 000000000..c665a08ae --- /dev/null +++ b/container/oci-image/tests/docker_garbage_test.py @@ -0,0 +1,130 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Garbage cleanup preserves live references and retained cluster resources.""" + +import contextlib +import io +import json +from pathlib import Path +import sys +import unittest +from unittest.mock import MagicMock + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from docker_garbage import collect +from docker_local_state import OWNER +from runtime import OWNER as HOST_OWNER + + +class DockerGarbageTest(unittest.TestCase): + def setUp(self): + self.docker = MagicMock() + self.runtime = dict(daemon='local', context='default', endpoint='unix:///docker.sock') + self.docker.identity.return_value = self.runtime + self.items = dict(container={}, network={}, volume={}) + self.used_volumes = set() + self.removed = [] + self.docker.objects.side_effect = lambda kind, *args: list(self.items[kind].values()) + self.docker.call.side_effect = self.call + + def call(self, *args): + if args[1] == 'inspect': + return json.dumps([self.items[args[0]][args[2]]]) + if args[:2] == ('ps', '-aq'): + return 'live-user' if args[-1].removeprefix('volume=') in self.used_volumes else '' + if args[:2] == ('builder', 'prune'): + return 'Total reclaimed space: 0B' + self.removed.append(args) + return '' + + def container(self, identity, status='exited', owner=None): + item = dict(Id=identity, State=dict(Running=status in ('running', 'paused', 'restarting'), + Status=status), Config=dict(Labels={OWNER: owner})) + self.items['container'][identity] = item + return item + + def volume(self, name, owner=None): + self.items['volume'][name] = dict(Name=name, Labels={OWNER: owner}) + + def network(self, name, attached=False, owner=None): + self.items['network'][name] = dict(Id=name, Name=name, Labels={OWNER: owner}, + Containers=dict(live={}) if attached else {}) + + def collect(self, records=(), hosts=(), **kwargs): + with contextlib.redirect_stdout(io.StringIO()): + collect(self.docker, records, hosts, runtime=self.runtime, **kwargs) + + def test_removes_stopped_containers_before_unused_named_and_anonymous_volumes(self): + for status in ('created', 'exited', 'dead', 'running', 'paused', 'restarting'): + self.container(status, status) + self.volume('anonymous') + self.volume('named-data') + self.volume('running-data') + self.used_volumes.add('running-data') + self.network('old-bridge') + self.network('live-bridge', attached=True) + self.collect() + self.assertEqual(self.removed, [('rm', 'created'), ('rm', 'exited'), ('rm', 'dead'), + ('network', 'rm', 'old-bridge'), ('volume', 'rm', 'anonymous'), + ('volume', 'rm', 'named-data')]) + self.docker.call.assert_any_call('builder', 'prune', '--all', '--force') + self.assertFalse(any(item.args[0] in ('stop', 'kill') for item in self.docker.call.call_args_list)) + + def test_retained_cluster_keeps_stopped_members_and_detached_resources(self): + records = [dict(cluster=name, owner=name, nodes=1) for name in ('first', 'second')] + for record in records: + self.container(record['cluster'], owner=record['owner']) + self.volume('crowdb-' + record['cluster'] + '-1-data') + self.volume('extra-' + record['cluster'], owner=record['owner']) + self.network('crowdb-' + record['cluster']) + host = self.container('host-stopped') + host['Config']['Labels'] = {HOST_OWNER: 'host-owner'} + self.collect(records, [dict(owner='host-owner')]) + self.assertEqual(self.removed, []) + + def test_keep_data_preserves_all_volumes_but_removes_stopped_containers(self): + self.container('old') + self.volume('unused-data') + self.collect(delete_data=False) + self.assertEqual(self.removed, [('rm', 'old')]) + self.assertFalse(any(item.args[0] == 'volume' for item in self.docker.call.call_args_list)) + + def test_default_networks_survive_even_if_reported_as_unused(self): + for name in ('bridge', 'host', 'none'): + self.network(name) + self.collect() + self.assertEqual(self.removed, []) + + def test_restarted_container_is_skipped_after_fresh_inspection(self): + old = self.container('restarted') + self.docker.objects.side_effect = lambda kind, *args: [old] if kind == 'container' else [] + self.items['container']['restarted'] = dict(old, State=dict(Running=True, Status='running')) + self.collect() + self.assertEqual(self.removed, []) + + def test_docker_reference_race_failure_is_reported_without_force_or_cache_prune(self): + self.volume('newly-used') + original = self.call + + def call(*args): + if args[:2] == ('volume', 'rm'): + raise RuntimeError('volume is in use') + return original(*args) + + self.docker.call.side_effect = call + with self.assertRaisesRegex(RuntimeError, 'volume is in use'): + self.collect() + self.docker.call.assert_any_call('volume', 'rm', 'newly-used') + self.assertFalse(any(item.args[0] == 'builder' for item in self.docker.call.call_args_list)) + + def test_context_change_before_removal_has_no_effects(self): + self.container('old') + self.docker.identity.side_effect = [self.runtime, dict(daemon='foreign')] + with self.assertRaisesRegex(ValueError, 'endpoint/context changed'): + self.collect() + self.assertEqual(self.removed, []) + self.assertFalse(any(item.args[0] == 'builder' for item in self.docker.call.call_args_list)) + + +if __name__ == '__main__': + unittest.main() diff --git a/container/oci-image/tests/docker_local_test.py b/container/oci-image/tests/docker_local_test.py index 1f4bb5eb7..858062fcd 100644 --- a/container/oci-image/tests/docker_local_test.py +++ b/container/oci-image/tests/docker_local_test.py @@ -10,6 +10,7 @@ import unittest from unittest.mock import MagicMock, patch import uuid +import yaml sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from docker_local_cleanup import clean, image_garbage, selected @@ -19,7 +20,7 @@ class DockerLocalTest(unittest.TestCase): def setUp(self): - self.record = {"version": 1, "cluster": "first", "owner": str(uuid.uuid4()), + self.record = {"version": 2, "cluster": "first", "owner": str(uuid.uuid4()), "runtime": {"daemon": "owned", "context": "default", "endpoint": "unix:///owned"}, "image_id": "sha256:retained", "image": "crowdb-node:dev", "nodes": 1, "port_base": 9090, "cpus": 1, "memory_mib": 1536} @@ -85,6 +86,23 @@ def test_changed_daemon_rejects_before_cleanup_effects(self): clean(self.docker, self.record) self.docker.call.assert_not_called() + def test_running_cluster_is_rejected_before_any_effect_in_stopped_only_mode(self): + self.docker.owned.return_value = [dict(State=dict(Running=False)), dict(State=dict(Running=True))] + with self.assertRaisesRegex(ValueError, 'cluster is running'): + clean(self.docker, self.record, stop_running=False) + self.docker.call.assert_not_called() + + def test_container_restarted_during_cleanup_is_never_stopped(self): + stopped = dict(Id='owned', State=dict(Running=False)) + restarted = dict(stopped, Name='/crowdb-first-1', Config=dict(Labels=self.labels), + State=dict(Running=True)) + self.docker.owned.side_effect = [[stopped], [stopped]] + self.docker.call.return_value = json.dumps([restarted]) + with self.assertRaisesRegex(ValueError, 'container restarted'): + clean(self.docker, self.record, stop_running=False) + self.assertEqual([item.args for item in self.docker.call.call_args_list], + [('container', 'inspect', 'owned')]) + def test_delete_data_rejects_a_volume_referenced_by_another_container(self): volume = {"Name": "crowdb-first-1-data", "Labels": self.labels} self.docker.owned.side_effect = lambda record, kind: [volume] if kind == "volume" else [] @@ -109,23 +127,41 @@ def test_changed_container_inputs_are_rejected_before_resume(self): item = {"Name": "/crowdb-first-1", "Image": record["image_id"], "Config": {"Labels": self.labels, "Env": environment}, "HostConfig": {"NanoCpus": 1_000_000_000, "Memory": 1536 * 1024 * 1024, "NetworkMode": "crowdb-first", "PortBindings": { - f"{port}/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(port)}] - for port in range(9090, 9094)}}, + f"{port}/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(9090+offset)}] + for offset, port in enumerate((9090, 16000, 17000, 18000))}}, "Mounts": [{"Destination": "/opt/crowdb/data", "Name": "owned-volume", "Type": "volume", "RW": True}]} verify_container(item, record, 1, "owned-volume", "crowdb-first") + legacy = dict(item, HostConfig=dict(item["HostConfig"], PortBindings={ + f"{port}/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(port)}] + for port in range(9090, 9094)})) + verify_container(legacy, dict(record, version=1), 1, "owned-volume", "crowdb-first") + with self.assertRaisesRegex(ValueError, "endpoints"): + verify_container(legacy, record, 1, "owned-volume", "crowdb-first") for field, value in (("Image", "foreign"), ("Mounts", []), ("HostConfig", {})): with self.subTest(field=field), self.assertRaises((ValueError, KeyError)): verify_container(dict(item, **{field: value}), record, 1, "owned-volume", "crowdb-first") + def test_compose_examples_publish_the_standard_node_listeners(self): + for filename in ("compose.yaml", "compose-three.yaml"): + document = yaml.safe_load((Path(__file__).resolve().parents[1] / filename).read_text()) + for service in document["services"].values(): + self.assertEqual([port.split("}:", 1)[1] for port in service["ports"]], + ["9090", "${CROWDB_S3_CONTAINER_PORT:-16000}", + "${CROWDB_ICEBERG_CONTAINER_PORT:-17000}", + "${CROWDB_DATASET_CONTAINER_PORT:-18000}"]) + def test_receipts_are_immutable_scoped_and_reject_symlinks(self): with tempfile.TemporaryDirectory() as temporary: directory = Path(temporary) seal(self.record, directory) self.assertEqual(load("first", directory), self.record) + legacy = dict(self.record, cluster="legacy", version=1) + seal(legacy, directory) + self.assertEqual(load("legacy", directory), legacy) self.assertEqual(selected("first", False, directory), [self.record]) other = dict(self.record, cluster="second", owner=str(uuid.uuid4())) seal(other, directory) - self.assertEqual(len(selected(None, True, directory)), 2) + self.assertEqual(len(selected(None, True, directory)), 3) with self.assertRaises(ValueError): seal(dict(self.record, owner=str(uuid.uuid4())), directory) (directory / "foreign.json").symlink_to(directory / "first.json") diff --git a/container/oci-image/tests/host_cleanup_test.py b/container/oci-image/tests/host_cleanup_test.py index 883a2b978..3ce6e01ed 100644 --- a/container/oci-image/tests/host_cleanup_test.py +++ b/container/oci-image/tests/host_cleanup_test.py @@ -20,6 +20,20 @@ def test_exact_owner_is_stopped_then_removed_without_data_effects(self): clean(self.record) self.assertEqual(self.runtime.effects, [('stop', 'owned-container'), ('rm', 'owned-container')]) + def test_stopped_only_cleanup_never_stops_a_running_host(self): + self.runtime.current = fixture.container(self.record) + with patch('host_cleanup.HostRuntime', return_value=self.runtime): + with self.assertRaisesRegex(ValueError, 'host container is running'): + clean(self.record, stop_running=False) + self.assertEqual(self.runtime.effects, []) + + def test_stopped_only_cleanup_removes_an_exited_host(self): + self.runtime.current = fixture.container(self.record) + self.runtime.current['State']['Running'] = False + with patch('host_cleanup.HostRuntime', return_value=self.runtime): + clean(self.record, stop_running=False) + self.assertEqual(self.runtime.effects, [('rm', 'owned-container')]) + def test_foreign_owner_or_daemon_never_receives_effects(self): for changed_daemon in (False, True): self.runtime.current = fixture.container(self.record) diff --git a/container/oci-image/tests/host_inputs_test.py b/container/oci-image/tests/host_inputs_test.py index 05400046f..77205ebbd 100644 --- a/container/oci-image/tests/host_inputs_test.py +++ b/container/oci-image/tests/host_inputs_test.py @@ -16,6 +16,14 @@ class HostInputsTest(unittest.TestCase): + def test_port_offset_shifts_every_host_listener_for_additional_instances(self): + shifted = run_node.shifted_ports(1) + self.assertEqual(shifted[0], 2223) + self.assertIn(10001, shifted) + self.assertIn(10101, shifted) + with self.assertRaisesRegex(ValueError, "beyond port 65535"): + run_node.shifted_ports(65535) + def test_runtime_logs_preserve_stderr_without_polluting_machine_readable_inspection(self): with patch('runtime.subprocess.run', return_value=SimpleNamespace( returncode=0,stdout='{"status":"running"}',stderr='startup failed on stderr')): diff --git a/container/oci-image/tests/install_deps_test.py b/container/oci-image/tests/install_deps_test.py index 864d460df..6e08696e3 100644 --- a/container/oci-image/tests/install_deps_test.py +++ b/container/oci-image/tests/install_deps_test.py @@ -27,13 +27,25 @@ def execute(self, platform, fail_setup=False, installed_ui=False, fail_ui=False, log = root / 'commands' for name in ('git', 'cargo', 'pixi', 'uname', 'npm'): stub = binaries / name - stub.write_text('#!/bin/sh\n' - + ('printf "%s\\n" "$TEST_PLATFORM"\n' if name == 'uname' else - f'printf "%s\\n" "{name} $*" >> "$TEST_COMMANDS"\n' - 'case "$*" in\n' - ' *--setup-apparmor*) exit "$TEST_SETUP_EXIT" ;;\n' - f' ci) exit "$TEST_UI_EXIT" ;;\n' - 'esac\n')) + stub_lines = ['#!/bin/sh\n'] + if name == 'uname': + stub_lines.append('printf "%s\\n" "$TEST_PLATFORM"\n') + else: + if name == 'pixi': + stub_lines.extend([ + 'if [ "$1" = completion ]; then\n', + ' printf "%s\\n" \'complete -W "stub-task" pixi\'\n', + ' exit 0\n', + 'fi\n', + ]) + stub_lines.extend([ + f'printf "%s\\n" "{name} $*" >> "$TEST_COMMANDS"\n', + 'case "$*" in\n', + ' *--setup-apparmor*) exit "$TEST_SETUP_EXIT" ;;\n', + f' ci) exit "$TEST_UI_EXIT" ;;\n', + 'esac\n', + ]) + stub.write_text(''.join(stub_lines)) stub.chmod(0o755) environment = dict(os.environ, HOME=str(root), PIXI_PROJECT_ROOT=str(root), TEST_PLATFORM=platform, TEST_COMMANDS=str(log), TEST_SETUP_EXIT='1' if fail_setup else '0', @@ -46,6 +58,22 @@ def execute(self, platform, fail_setup=False, installed_ui=False, fail_ui=False, break return result, log.read_text().splitlines(), (root / '.bashrc').read_text() + def assert_completion_loads_in_new_bash(self, bashrc): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / '.bashrc').write_text(bashrc) + binaries = root / 'bin' + binaries.mkdir() + pixi = binaries / 'pixi' + pixi.write_text('#!/bin/sh\n' + 'printf \'complete -W "stub-task" pixi\\n\'\n') + pixi.chmod(0o755) + environment = dict(os.environ, HOME=str(root), PATH=str(binaries) + os.pathsep + os.environ['PATH']) + result = subprocess.run(['bash', '-ic', 'complete -p pixi'], env=environment, + text=True, capture_output=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn('stub-task', result.stdout) + def test_linux_sets_up_host_before_ui_and_cargo_dependencies(self): result, commands, _ = self.execute('Linux x86_64') self.assertEqual(result.returncode, 0, result.stdout + result.stderr) @@ -83,6 +111,8 @@ def test_installs_completion_without_overwriting_existing_shell_configuration(se self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertTrue(bashrc.startswith(original)) self.assertIn('source <(pixi completion --shell bash)', bashrc) + self.assertIn('CROWDB Pixi Bash completion', bashrc) + self.assertIn("source /", result.stdout) def test_repeated_dependency_setup_does_not_duplicate_completion(self): result, _, bashrc = self.execute('Linux x86_64', repeats=2) @@ -97,6 +127,11 @@ def test_existing_completion_is_preserved_without_adding_another_block(self): self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertEqual(bashrc, original) + def test_new_bash_loads_completion_block(self): + result, _, bashrc = self.execute('Darwin arm64') + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assert_completion_loads_in_new_bash(bashrc) + if __name__ == '__main__': unittest.main() diff --git a/container/oci-image/tests/task_entrypoints_test.py b/container/oci-image/tests/task_entrypoints_test.py index 373fbcb14..7e1cd3b6e 100644 --- a/container/oci-image/tests/task_entrypoints_test.py +++ b/container/oci-image/tests/task_entrypoints_test.py @@ -79,7 +79,7 @@ def test_unknown_clean_options_fail_before_effects(self): def test_manifest_exposes_unified_tasks_and_has_no_stale_dependency_names(self): tasks = tomllib.loads((ROOT / 'pixi.toml').read_text())['tasks'] - self.assertTrue({'install-deps', 'clean', 'build'} <= tasks.keys()) + self.assertTrue({'install-deps', 'clean', 'build-native', 'build'} <= tasks.keys()) self.assertFalse({'install-ui-deps', 'clean-env', 'build-cpp', 'build-tests'} & tasks.keys()) for name, task in tasks.items(): if isinstance(task, dict): diff --git a/container/single-node-container/README.md b/container/single-node-container/README.md index 403ac9543..b0ba77b41 100644 --- a/container/single-node-container/README.md +++ b/container/single-node-container/README.md @@ -61,9 +61,11 @@ remain independent of public origins. All members of one local cluster share the accepted client credential set. Retain each member's own data volume across stop and image replacement. -Default `oci-clean` removes all recorded clusters, their unused owned bridges -and owned data. Use `--cluster NAME` to select one cluster and `--keep-data` to -retain its volumes and restart receipts. +Default `oci-clean` preserves running clusters and removes stopped clusters, +their owned data, and unused Docker containers, networks, volumes and build +cache. Use `--cluster NAME` to explicitly stop and remove one cluster, or +`--all` to stop and remove all recorded clusters. Use `--keep-data` to retain +volumes and restart receipts. See the [cleanup contract](../oci-image/README.md#local-cluster-shortcuts). Ordinary `docker compose down` also preserves its named volumes. ## Tested S3 client recipes diff --git a/container/single-node-container/entrypoint.sh b/container/single-node-container/entrypoint.sh index 46f84cb22..6c65fa3fc 100644 --- a/container/single-node-container/entrypoint.sh +++ b/container/single-node-container/entrypoint.sh @@ -60,18 +60,29 @@ if test "$port_offset" -gt 0; then offset_templates=$offset_root/templates mkdir -p "$offset_templates" cp "$profile" "$offset_root/profile.toml" - cp /opt/crowdb/etc/templates/*.toml "$offset_templates/" + if test -d /opt/crowdb/etc/templates; then + for template in /opt/crowdb/etc/templates/*.toml; do + test -f "$template" || continue + cp "$template" "$offset_templates/" + done + fi # The preview profile is intentionally self-contained. Shift every # declared listener and endpoint together so host-network containers can # coexist while retaining the same relative service layout. - for base in 9090 9091 9092 10000 10100 10101 11000 11100 11200 12100 12200 13000 15100 15200; do + for base in 19000 18000 17000 16000 15200 15100 14000 13000 12200 12100 12000 11200 11100 11000 10101 10100 10000 9095 9094 9093 9092 9091 9090 7401 7400 7301 7300 7200 7102 7101 7100; do next=$((base + port_offset)) - sed -i -E "s/(^|[^0-9])${base}([^0-9]|$)/\\1${next}\\2/g" \ - "$offset_root/profile.toml" "$offset_templates"/*.toml + sed -i -E "s/(^|[^0-9])${base}([^0-9]|$)/\\1${next}\\2/g" "$offset_root/profile.toml" + if test -d "$offset_templates"; then + for template in "$offset_templates"/*.toml; do + test -f "$template" || continue + sed -i -E "s/(^|[^0-9])${base}([^0-9]|$)/\\1${next}\\2/g" "$template" + done + fi done sed -i "s#/opt/crowdb/etc/templates#$offset_templates#g" "$offset_root/profile.toml" profile="$offset_root/profile.toml" fi +ssh_port=$((2222 + port_offset)) /opt/crowdb/bin/crowdb-monitor validate "$profile" if test -n "${CROWDB_STARTUP_MODE:-}"; then @@ -149,7 +160,7 @@ chmod 600 /opt/crowdb/data/ssh/authorized_keys chown -R crowdb:crowdb /opt/crowdb/data/ssh /opt/crowdb/run if ! test -e /opt/crowdb/.ssh; then ln -s /opt/crowdb/data/ssh /opt/crowdb/.ssh; fi cat > /opt/crowdb/run/sshd_config < its own resource menu. | | +| | Deleted Object disappears on success, then list refreshes. | | +----------------------------+--------------------------------------------------------------+------------------------------+ ``` @@ -1488,7 +1564,12 @@ CRUD merely to match another domain. Dataset lifecycle support is defined in §2 at 10/30 s retain slow/very-slow reports. Compare per-test baselines; investigate more than 2× rather than hiding time with retries or larger deadlines. - **TEST-05:** One worker owns the mutable runtime; installed system browser, - no automatic browser installation. Default assertion deadline is 3 s and + no automatic browser installation. Auxiliary-service dependency convergence + may wait up to 10 s for Chunk-KV range activation and matching catalog/grant + authority. After 3 s it records the latest readiness response; it also records + the final response and elapsed time. This prerequisite does not change UI + response or process-observation assertion budgets. + Default assertion deadline is 3 s and election 10 s, polling every 100 ms. No fixed sleeps. Native large multipart and SF=1 loader acceptance run separately from ordinary UI iteration; normal smoke data remain small while preserving multi-Strip and pagination cases. diff --git a/doc/design/deploy/design-crowdb-deploy.md b/doc/design/deploy/design-crowdb-deploy.md index caad22017..18038b3c5 100644 --- a/doc/design/deploy/design-crowdb-deploy.md +++ b/doc/design/deploy/design-crowdb-deploy.md @@ -96,6 +96,11 @@ numeric node ID through Group 0 and durably records the mapping. Allocation and mapping publication are conditional and idempotent under concurrent admission; IDs cannot collide or be silently reassigned. Endpoint and rack changes do not change either identity. Duplicate live UUID claims are identity conflicts. +Confirmed admission retains the Web listener proved over authenticated SSH. +Cluster observation uses that listener for each UUID, including containers with +shared host addresses and different port offsets. Retained default-port records +keep their existing serialized identity; each monitor republishes its own current +listener through the existing confirmed-node update operation. ## 3. Invariants @@ -255,7 +260,7 @@ requires an explicit recovery/decommission procedure, not an automatic timeout. 1. Any UI collects compatible candidates and the operator's rack assignments. 2. The operator selects initial Group-0 voting members, or Auto join selects - the explicitly selected one or three authenticated eligible candidates in + the explicitly selected one, three or five authenticated eligible candidates in discovery-UUID order. Candidate count is not voting-member count; application nodes need not all vote. 3. Submit the bootstrap identity, immutable initial manifest and initial data @@ -311,20 +316,30 @@ Production requires configured rack assignments and access. Missing Production credentials remain Needs access, without a Local fallback. Multiple bound cluster UUIDs require explicit target selection rather than an arbitrary advertisement. -The accepted initial voter count is one or three. An incomplete three-node set -remains pending; an observed overfull fresh set reports conflict before acceptance. +The accepted initial voter count is one, three or five. An incomplete selected +set remains pending; an observed overfull fresh set reports conflict before acceptance. Eligible UUID ordering selects the coordinator within that set, while monitor and KV-server accepted-manifest fencing prevent conflicting execution. Fixed voters never expand from later advertisements. Turning Auto join off stops new scheduling while already accepted monitor work continues. After Group-0 publication, one committed default preparation intent owns the -ordinary metadata group, replica mapping and shared access credentials. Authorized -workers resume that intent across Web/coordinator loss. Each monitor interleaves +ordinary metadata group, replica mapping and shared access credentials. The +replica mapping covers every fixed initial voting member; each receives its own +initial storage binding. Authorized workers resume that intent across +Web/coordinator loss. Each monitor interleaves disk preparation with dependent service startup; DiskIO may start before Online because its ownership proof enables Online. Explicit later service stops or group retirement supersede historical setup and prevent automatic recreation. +Background service recovery observes readiness once per pass, retaining the +same live child and durable launch attempt until readiness succeeds. Explicit +service commands retain their bounded initial startup wait. Local read-only +health probes run concurrently with their individual deadlines, and publication +rechecks actual owned process liveness and PID before reporting health. Slow +startup or several unavailable listeners must remain visible as unhealthy; +publishing a fresh timestamp cannot substitute for those observations. + The versioned deployment policy is durable and independent of UI permissions. Local and Production inputs cannot replace one another through an environment change. Initial voter count, storage, listeners and scope are sealed before node @@ -363,6 +378,15 @@ SSH connection/authentication has a ten-second bound; bidirectional proof has a twenty-second bound. Passwords are discarded, sessions are reused during setup, and only public keys cross nodes. +Each Web process sampler owns one read-only SSH connection per bound remote +node. Process observations retain their one-second cadence and three-second +deadline; connection reuse does not extend observation freshness. The sampler +checks the returned node and complete bootstrap identity on every response. +Connections are discarded after a command error, cancelled read, binding or +node configuration change, and are reauthenticated at least once per minute. +They use the provisioned node key and existing strict host-key verification, +without password fallback or sharing a session with management mutations. + ### 6.4 Authenticated member address recovery Node identity survives changes to management and data addresses. Each accepted @@ -395,8 +419,12 @@ trust or automatic key replacement for an existing identity. Recovery proceeds independently of Group-0 availability: -1. mDNS/DNS-SD or a configured seed reveals a candidate's current management - endpoint. Its advertised UUIDs and node ID are untrusted lookup hints. +1. mDNS/DNS-SD, a configured seed or retained accepted contact supplies a + candidate management address. Retained contacts remain probe hints when + multicast or Group 0 is unavailable. Advertised identities and addresses + never authorize a route without fresh proof. Fresh discovery hints receive + the bounded probe budget before retained addresses, so a full old-address + cache cannot suppress a member's changed IP. 2. The monitor selects an already accepted member binding and opens SSH to that address with the expected host key, rather than trusting the IP's previous occupant or accepting a new key. SSH proves possession of the private key. @@ -423,6 +451,16 @@ Recovery proceeds independently of Group-0 availability: conditional new service operations; later stop/delete commands win. Public client URIs are independent of these internal addresses. +The contact worker checks retained retirement and accepted identity before +probing peers. After authenticating recovered routes, address publication +checks current authority through those routes before optional confirmation +refreshes retained contacts. Publication and confirmation share a three-second +authority budget; SSH proofs retain their ten-second bound. Unreachable old +authority contacts must not consume the budget before recovered addresses can +be published, nor add another full wait that expires otherwise valid routes at +their fifteen-second freshness fence. Current authority still fences removed +members and conditional service operations before publication. + This permits all members to restart with changed IPs at once. Neither stable DNS names nor fixed IPs nor a complete peer-address list is required. Multicast must reach the selected management scope; across discovery boundaries, seed discovery @@ -578,6 +616,16 @@ process and before starting its replacement. A superseded operation or unavailab authority rejects execution. Recovery reads current desired intents through the same authority path and does not execute a retained command solely from disk. +An exact completed KV Stop/Delete can return its retained receipt while that +child remains stopped. The receipt carries the bootstrap identity under which +the operation was authorized; it must match the current accepted local binding +and complete intent, with no live replacement, cancellation or retirement. +This reports a prior effect without executing another one. It must not require +the Group-0 leader that the operation already stopped: background recovery may +finish the intent before its identical SSH request reaches the executor. +Legacy receipts without that bootstrap proof still require current authority. +Incomplete, superseded and new operations retain every authority fence above. + Admission commands similarly match the current cluster publication and UUID, numeric ID, operation ID and cancellation state. Local durable cancellation and retirement markers reject delayed commands after cleanup or restart. Long-running diff --git a/doc/design/deploy/design-crowdb-oci-image.md b/doc/design/deploy/design-crowdb-oci-image.md index 4435269cd..a216a22d8 100644 --- a/doc/design/deploy/design-crowdb-oci-image.md +++ b/doc/design/deploy/design-crowdb-oci-image.md @@ -304,18 +304,22 @@ file hashes. Resume accepts the same inputs and exact owned container, or recreates that container against the retained data. A foreign same-name object, changed daemon or changed configuration fails before start/stop effects. Application UUID binding is separate and can be sealed only after observing -that cluster. One cleanup entry point selects all recorded bridge and host-network -clusters by default, or one explicitly named cluster. Cleanup verifies the same -ownership and removes containers, unused owned bridges, node data and matching -launch/application receipts. Bridge data is held in owned Docker volumes. Host -data deletion requires the retained storage declaration to match accepted media +that cluster. Default cleanup selects stopped recorded bridge and host-network +clusters and preserves a running cluster's containers, data and restart receipts. +Explicit cluster or all-cluster teardown can stop the selected containers. +Cleanup verifies the same ownership and removes containers, unused owned bridges, +node data and matching launch/application receipts. Bridge data is held in owned +Docker volumes. Host data deletion requires the retained storage declaration to match accepted media and rejects symlinked roots, nested mounts, overlapping recorded data directories and bind mounts referenced by another container in the selected runtime. Supplied disk files, raw devices and external policy/password inputs are preserved; cleanup never wipes their contents. Retaining data and restart receipts is an explicit cleanup option. Failures leave ownership retryable until data deletion -and target-binding removal complete. Containerd-only cleanup requires no Docker -daemon or CLI binary. +and target-binding removal complete. Unscoped Docker cleanup also removes stopped +containers without receipts, unreferenced named and anonymous volumes, unused +networks and unused build cache. Retained cluster resources remain protected; +non-force runtime removal rejects concurrent restarts or new references. +Containerd-only cleanup requires no Docker daemon or CLI binary. Explicit monitor liveness/readiness probes report node health. A dedicated systemd unit restarts the containerd daemon, and its restart plugin applies the diff --git a/doc/design/diskdb/design-crowdb-diskdb.md b/doc/design/diskdb/design-crowdb-diskdb.md index 8b8ee84d5..2dc994d4e 100644 --- a/doc/design/diskdb/design-crowdb-diskdb.md +++ b/doc/design/diskdb/design-crowdb-diskdb.md @@ -367,6 +367,11 @@ disk-group's bound data group. `diskdb-ownership` monitor assigns bound disk-groups to live, fencing-capable DiskDB instances. Missing or dead owners are replaced; count balancing moves at most one healthy group per tick. Updates compare the owner record revision. + Before a live capable owner's one-hour placement lease expires, the monitor + renews it through the same revision-checked publication path without changing + the instance or process epoch. The DiskDB observes the new map generation and + reacquires its data-group fence. Fresh leases are not rewritten each tick; + renewals defer count balancing until the next snapshot. - **Binding map** (`/hw/dg_bind/...`) — the console recommends the ordinary KV group with the fewest bound DiskGroups cluster-wide and allows an explicit override at creation and publishes the binding, hardware record, and node membership atomically. Missing selections, Group 0, and unknown diff --git a/doc/design/diskio/design-crowdb-diskio.md b/doc/design/diskio/design-crowdb-diskio.md index 5abbf20fd..a69986ab2 100644 --- a/doc/design/diskio/design-crowdb-diskio.md +++ b/doc/design/diskio/design-crowdb-diskio.md @@ -754,6 +754,12 @@ client's references, while the final in-flight reference controls when its C++ connection is closed. Selection and route/status snapshots use immutable RCU state and atomics, so request routing does not acquire a caller lock. +Readiness uses the priority group for bounded 4096-byte reads of existing disk +headers. Normal bulk responses and readiness headers therefore have independent +TCP response ordering while sharing the same validated disk identity and route. +Priority selection does not change physical read semantics, error propagation, +route refresh rules or the caller's total deadline. + The internal wire transport retains the caller's owned `Bytes` allocation in the RPC buffer until completion. This avoids a client-side `Bytes` to `Vec` payload copy. On the server, block-size-one and already aligned payloads can diff --git a/doc/design/kv/design-crowdb-kv-group0.md b/doc/design/kv/design-crowdb-kv-group0.md index 0c75815f4..7d02c0e18 100644 --- a/doc/design/kv/design-crowdb-kv-group0.md +++ b/doc/design/kv/design-crowdb-kv-group0.md @@ -396,6 +396,16 @@ Vec`; kv-server carries `hosted_stores`, `hosted_groups`, (`register_diskdb`/`heartbeat_diskdb`/`read_all_diskdb_instances`) and kv-server wrappers delegate to the generic methods. +DiskIO physical identity and configured hardware scope are immutable once +accepted. Physical identity is pinned with instance-revision CAS before scope +publication. The first configured scope requires an existing Rack record and +is published atomically with an advance of that Rack's revision, fencing +concurrent disk membership changes. Revalidate the instance after reading the +Rack revision so another first-scope publication cannot be overwritten. If the +Rack is absent, retain the physical pin and reject scope publication; an +instance-only write cannot substitute for the Rack fence. Ordinary heartbeats +with the same accepted scope update liveness without advancing the Rack. + ### 4.2 Services registered - **diskdb** — `/srv/diskdb/`. Heartbeat includes diff --git a/doc/design/kv/design-crowdb-kv-leader-election.md b/doc/design/kv/design-crowdb-kv-leader-election.md index 3e4b2b28d..5566f47aa 100644 --- a/doc/design/kv/design-crowdb-kv-leader-election.md +++ b/doc/design/kv/design-crowdb-kv-leader-election.md @@ -215,6 +215,10 @@ While its lease is valid, a leader may serve linearizable reads from local state The first assumption is enforced by: - Acceptors **promise not to vote for any other candidate** for at least `lease_duration` after granting a heartbeat (this is the Raft "PreVote + lease" pattern). +- The leader rejects both PreVote and RequestVote while its quorum-renewed + effective lease remains valid, even after the original self-vote lockout + expires. Renewing this protection uses the existing atomic lease deadline; + a lagging minority cannot recruit the healthy leader as its extra voter. - The leader treats its lease as expired at `lease_grant_time + lease_duration - max_clock_skew` for safety (see §6.3). ### 6.2 Lease grant and renewal diff --git a/doc/design/kv/design-crowdb-kv-rpc-client.md b/doc/design/kv/design-crowdb-kv-rpc-client.md index d00178c22..21b0ca707 100644 --- a/doc/design/kv/design-crowdb-kv-rpc-client.md +++ b/doc/design/kv/design-crowdb-kv-rpc-client.md @@ -179,6 +179,17 @@ source. `conn_for` derives the client-facing crowdb-rpc port from the base port via the `KV_CLIENT_RPC_BASE` offset. Pool selection returns its generation, and retryable errors invalidate only that generation. +Callers with concurrent bulk scans and latency-bounded point reads may opt +into `with_isolated_point_reads`, or construct `CrowdbKvClient` with +`new_with_isolated_point_reads`. Get replies then use one additional connection +per endpoint, sharing the existing RPC workers, request-ID source, completion +pool, tunables and deadlines. Scans and mutations retain their ordinary pool. +Errors invalidate the selected pool's exact generation; explicit reconnect +clears both pools. The default constructor retains its existing connection +behavior. ChunkDB enables this mode so actual metadata point reads do not wait +behind an incomplete background scan response. Isolation does not cache success +or convert rejection, transport failure or an expired caller deadline to Ready. + Unary methods (`send_put`, `send_get`, `send_delete`, `send_batch_write`, `send_scan`, `send_journal_scan`): build request flatbuffer → `rpc.call` → await → parse via `Ref` wrapper → map to the existing @@ -263,6 +274,15 @@ election wait and topology refresh rather than issuing an unbounded RPC loop. Scan and journal continuations reset this redirect history after a successful page. Conditional writes retain their original mutation identity on retries. +After a transport failure or leadership rejection without a usable hint, +discovery searches the target group across management seeds. A reachable seed +that repeats the rejected RPC endpoint cannot hide another member's replacement +route. Concurrent refreshes share the existing refresh gate and interval; a +newer cached route is reused without another fetch. If no replacement is known, +the operation retains its original bounded election wait/backoff and retry +budget. Discovery never establishes authority for a mutation; the destination +still applies its normal leadership and conditional-write checks. + `TopologyCache` publishes one immutable `TopologySnapshot`. Every group route contains its leader, replica endpoints, read cursor, endpoint statistics, and write-slot high-watermark. A refresh builds a complete replacement snapshot; diff --git a/doc/design/kv/design-crowdb-kv-slot.md b/doc/design/kv/design-crowdb-kv-slot.md index da3130fc4..29b010a89 100644 --- a/doc/design/kv/design-crowdb-kv-slot.md +++ b/doc/design/kv/design-crowdb-kv-slot.md @@ -199,22 +199,44 @@ The `ballot_round` field in the ChosenNotice flatbuffer enables this verificatio When a follower detects a gap (missing or stale slot in the chosen range), it sends a unary `FetchGap(slot)` request to the leader. The leader: -- Has the chosen value locally → replies with the full entry (payload + ballot + term). -- Does not have it → runs classic Paxos (`repair_once`, §9) to resolve the slot, then replies with the resolved value (or NoOp). - -On receipt, the follower overwrites any stale accepted value with the chosen value, updates its chosen frontier, and wakes the apply loop. FetchGap is bounded by `MAX_INFLIGHT_FETCHGAP` (default 16) to avoid flooding the leader. +- Has completed leader recovery and retains the value within its contiguous + chosen prefix → replies with the full entry (payload + ballot + term). +- Otherwise → returns pending/not found. Normal leader recovery or independent + gap repair must establish the chosen value before a later request can fetch it. + +On receipt, a durable follower first writes the original Accepted entry to its +WAL, then installs it, advances the chosen frontier and wakes the apply loop. +Historical entries retain their original election term; a newer local term +does not reject committed history. Failed persistence leaves the gap pending. +Learning a proven chosen value does not answer a new Accept request and must +not discard a later promise. It retains the original entry alongside that +higher promise; an already higher accepted ballot must carry the same value. +Replay rebuilds highest accepted values before restoring promise fences, so +the physical order of historical fetch and later promise records cannot lose +either durable fact. Accepted history alone is never a FetchGap chosen proof. +`MAX_INFLIGHT_FETCHGAP` (default 16) bounds the entire fetch/persist/install +operation, including error and cancellation paths. This replaces the previous leader-driven catch-up that ran inline in `run_heartbeat_round` (up to 64 synchronous `send_accept().await` calls per lagging follower). That inline catch-up could exceed `heartbeat_interval` and trigger spurious elections. The heartbeat round is now pure liveness + lease: one RPC round-trip, no catch-up work. ### 9A.4 Snapshot Fallback -If a follower's gap count exceeds `catchup_snapshot_threshold` (default `bulk_prepare_window` = 1024), the follower stops issuing FetchGap requests and logs a warning. The full snapshot-install path for running replicas is deferred; the threshold gate prevents FetchGap storms against the leader when a follower is severely lagging (e.g. after a long network partition). +If a follower's gap count exceeds `catchup_snapshot_threshold` (default 1024), +it records the large backlog and continues FetchGap under the same fixed +16-request limit. A missing automatic snapshot path must not disable catch-up. The bounded snapshot protocol is used only while bootstrapping a fresh, unpublished member. It does not replace the engine of a running follower and -is not invoked by this threshold. A live replacement requires a separate +is not invoked by this diagnostic threshold. A live replacement requires a separate atomic engine-generation publication design. +A recovered durable engine's gap-free resume watermark remains authoritative +when WAL reclamation removed the boundary record or all preceding segments. +Restore retains that complete chosen/applied prefix without capping it to the +remaining WAL tip. If the exact boundary term is unavailable, its historical +term is reported as zero (unknown); voting freshness still uses the durable +acceptor log tip. Higher retained entries replay after the snapshot prefix. + ### 9A.5 Apply Loop The follower's apply loop targets `max(known_commit_slot, last_chosen_slot)`. For each slot in the apply range: @@ -315,19 +337,29 @@ See [WAL garbage collection](design-crowdb-kv-wal.md#7-garbage-collection). ## 12. Tunables and Defaults -| Parameter | Default | Where it lives | -| --- | --- | --- | -| `max_inflight_proposals` | 32 | `PaxosConfig` (total semaphore permits) | -| `reject_on_inflight_window_full` | `false` | `PaxosConfig` (internal, not CLI-exposed) | -| `coalesce_max_keys` | 32 | `PaxosConfig` (max ops per coalesced batch; 0 = off) | -| `coalesce_drain_threshold` | 1 | `PaxosConfig` (skip drain at this inflight count) | -| `max_paxos_retries` | 3 | `PaxosConfig` (per-slot Phase-2 retries) | -| `max_slot_retries` | 3 | `PaxosConfig` (new-slot retries before giving up) | -| `retry_base_backoff_ms` | 5 | `PaxosConfig` (exponential backoff base) | -| `bulk_prepare_window` | 1024 | `PxElectionConfig` (bulk Phase-1 batch size) | -| `catchup_snapshot_threshold` | 1024 | `PxElectionConfig` (gap count above which FetchGap is skipped in favor of snapshot fallback) | - -The bulk Phase-1 typically resolves open gaps in a single RTT. Steady-state gaps are closed one-at-a-time after each heartbeat round, so worst-case gap-clear time is bounded by `heartbeat_interval × gap_count`. Follower-side gaps (missing or stale chosen slots) are closed by FetchGap at the follower's own pace, bounded by `MAX_INFLIGHT_FETCHGAP` (default 16) concurrent requests. +| Parameter | Default | Where it lives | +| -------------------------------- | ------- | --------------------------------------------------------------------- | +| `max_inflight_proposals` | 32 | `PaxosConfig` (total semaphore permits) | +| `reject_on_inflight_window_full` | `false` | `PaxosConfig` (internal, not CLI-exposed) | +| `coalesce_max_keys` | 32 | `PaxosConfig` (max ops per coalesced batch; 0 = off) | +| `coalesce_drain_threshold` | 1 | `PaxosConfig` (skip drain at this inflight count) | +| `max_paxos_retries` | 3 | `PaxosConfig` (per-slot Phase-2 retries) | +| `max_slot_retries` | 3 | `PaxosConfig` (new-slot retries before giving up) | +| `retry_base_backoff_ms` | 5 | `PaxosConfig` (exponential backoff base) | +| `bulk_prepare_window` | 1024 | `PxElectionConfig` (bulk Phase-1 batch size) | +| `catchup_snapshot_threshold` | 1024 | `PxElectionConfig` (diagnostic threshold; bounded FetchGap continues) | + +Bulk Phase 1 recovers independent slots concurrently, with at most +`bulk_prepare_window` complete Prepare/Accept operations in flight. It starts +at the new leader's own contiguous frontier, adopts quorum values and opens +reads only after the entire observed prefix has resolved and applied. Tenure +cancellation prevents prepared slots from proceeding to Accept. + +Steady-state gap repair starts only after initial recovery has completed; +the two paths never compete for the same slot. Each background pass drains +its observed backlog independently of heartbeat cadence. Follower-side gaps +use at most `MAX_INFLIGHT_FETCHGAP` (default 16) concurrent requests, including +backlogs above the snapshot diagnostic threshold. --- diff --git a/doc/design/kv/design-crowdb-kv-wal.md b/doc/design/kv/design-crowdb-kv-wal.md index 8ecb3261b..b49b67819 100644 --- a/doc/design/kv/design-crowdb-kv-wal.md +++ b/doc/design/kv/design-crowdb-kv-wal.md @@ -177,7 +177,10 @@ Each individual `Accept` is small (tens of bytes overhead + payload). Per-record The durable operation is backend-specific: - **Filesystem file backend:** append with normal file writes, then `fdatasync` / equivalent durable flush. -- **Block-device backend:** issue aligned direct writes. If the device requires 4 KiB sectors, the WAL writer pads or read-modify-writes through the block backend's alignment planner; no filesystem `fdatasync` is involved. +- **Block-device backend:** issue aligned positional writes, using direct I/O + when selected. If the device requires 4 KiB sectors, the WAL writer pads or + read-modify-writes through the alignment planner. The real file-backed + implementation then calls `sync_data` before releasing durable acknowledgments. - **RAM / SCM / simulated backends:** use the backend's own durability contract. A volatile RAM backend is for tests only and may complete immediately; persistent-memory backends may require cache-line flush / fence semantics instead of filesystem sync. The acceptor code talks to `WalEngine::append_*` and does not branch on these backend details. @@ -249,6 +252,20 @@ enum with four variants: block device, or a tmpfs file makes no difference; in Unix, a block device *is* a file. +On the production multi-thread Tokio runtime, including a runtime with one +worker, synchronous block-device file and layout operations use Tokio's +[`block_in_place`](https://docs.rs/tokio/1.52.3/tokio/task/fn.block_in_place.html). +This allows independently spawned apply, RPC, and timer tasks +to run while the OS call blocks. The closure completes before returning, so +borrowed WAL buffers and the file remain live without an extra payload copy. +Write, durable flush, index publication, and acknowledgment keep their original +ordering and error behavior. This changes scheduling rather than durability, +timeouts, retries, or configured worker/resource limits. Current-thread and +non-runtime callers retain synchronous compatibility; they do not gain +concurrent task progress. The production path uses independently spawned WAL +workers on the ordinary multi-thread runtime, not a `LocalSet` or same-task +`join!` branches. + `BlockDevice` has two constructors selected by the `O_DIRECT` flag: - `BlockDevice::new()` — aligned (4K), buffered (no `O_DIRECT`). Default for @@ -360,6 +377,9 @@ The split matters: replay/restore are purely *local* (this node's WAL), but `Acc tail from a crash mid-write); later segments are still processed. 3. Rebuild per-`(group, slot)` acceptor state keeping the **highest-ballot** `Promised` / `Accepted` per slot (later/higher-ballot records win, Paxos rule). + Retain promises and accepted values independently while scanning; release + superseded payloads immediately. Segment locations, counts and election + state still account for every verified record. 4. `current_term` = max `term` across all records. 5. `voted_for` = the node from the latest `VoteGranted` whose `term == current_term`. This is election safety state, not just debug metadata: after crash, the node must not grant a second vote in the same term. Output: `ReplayResult { records, max_segment_id, current_term, voted_for }`. @@ -401,6 +421,13 @@ outright; see `../tree/design-crowdb-tree-engine.md`). `replay_group` → create `WalEngine` and resume `next_segment_id = max_segment_id + 1` → `restore_from_replay` → `set_wal` → wrap in `PxGroup`. +- Seed the live pipeline index with the verified replay index before starting + writers. Its slot locations and segment metadata cover retained history as + well as future appends, so GC can reclaim pre-restart segments. +- Initialize the WAL's local snapshot watermark from the freshly opened + engine's durable resume floor. Peer progress and the reclamation certificate + remain independent gates; local recovery does not authorize prefix removal. + The group also records a conservative local slot tip: ``` @@ -466,9 +493,12 @@ the node fails out of the group and rebuilds from peers via snapshot install. ### 6.7 Replay performance -Replay is sequential disk reads, parallelized across disks; WAL sizes stay small -because of GC (§7), so replays are typically sub-second. The dominant cost is -state machine snapshot load (if any), not WAL read. +Binary replay uses a 1 MiB read-ahead buffer for frame markers, format probes +and complete frames. A larger frame receives enough space for its own length. +Every frame still passes its normal CRC and format checks. Retained payloads +own their bytes so replacing an accepted record can release it independently +of neighboring records in the input buffer. Memory scales with the retained +highest-ballot state and index, rather than the number of replaced versions. --- @@ -493,6 +523,14 @@ contract is in [slot retention](design-crowdb-kv-slot.md#11-interaction-with-sna GC runs at **segment granularity**, not record granularity. A whole segment is unlinked from disk when *every* record in it has `slot < gc_slot`. This is cheap (one `unlink()` per segment) and avoids any in-place rewriting. +- Slot-based eligibility does not retire election state. Retain a segment + proving the highest term and the segment containing the latest vote at the + highest recorded vote term, even when their slot records are below the GC + watermark. A mixed vote/data segment follows the same rule. +- Derive these election fields from every verified record during replay and + update them as live segments are written. They are in-memory index metadata; + WAL record and segment formats remain unchanged. + ### 7.3 GC trigger GC is triggered by: diff --git a/doc/design/tree/design-crowdb-tree-chunk-storage.md b/doc/design/tree/design-crowdb-tree-chunk-storage.md index cf24f22ff..7960713ff 100644 --- a/doc/design/tree/design-crowdb-tree-chunk-storage.md +++ b/doc/design/tree/design-crowdb-tree-chunk-storage.md @@ -57,6 +57,26 @@ and a `ChunkPageRef` containing chunk identity, physical offset, logical length, and frame revision. Frame footers own payload integrity and expected chunk identity; high-cardinality reference entries do not duplicate their checksums. +Pending writes form a sparse byte overlay on one captured immutable manifest. +Starting a write does not materialize the published logical image or its holes. +Partial writes capture only their intersected published packs, retaining the +verified untouched bytes even if those mirrors become unavailable before commit. +Later overlapping writes replace earlier bytes; overlay fragments are bounded +to 64 KiB so replacing a small range cannot copy an arbitrarily large fragment. +Each fragment retains its full logical coverage while storing trailing zeros +implicitly. Zero padding therefore still overwrites base bytes and participates +in coverage checks without retaining an allocation-unit buffer per padded page. +Publication assembles one pack from verified base bytes and the overlay at a +time. Unchanged packs still undergo frame verification before reuse. Both +publication paths discard the overlay after commit, releasing its buffers. +The tree releases each encoded snapshot page as soon as its store write +completes, including asynchronous completion. It retains the page's publication +identity through the final anchor barrier, without retaining a second copy of +all bytes already accepted into the backend overlay. +Snapshot preparation retains logical encoded page bytes. It reserves aligned +addresses immediately but adds zero padding to the current page only at write +submission, avoiding an additional allocation-unit padding copy of every page. + Mapping-table slots do not store physical chunk offsets. Their tagged 64-bit page-reference words name a logical location whose covering pack is resolved through immutable segmented reference tables. Untagged words remain valid for @@ -147,6 +167,15 @@ generation and ownership epoch. Only that compare-and-publish operation makes the checkpoint visible. Stale, failed, or ambiguous work remains unreachable and is accounted as orphan data. +Once a tree has written an anchor, a failed final durability barrier leaves +that anchor pending under the existing snapshot serialization gate. Before +preparing another snapshot, synchronous and asynchronous callers retry the +pending barrier over the same staged image. Continued unavailability remains +a typed failure; no new snapshot data is written until the pending barrier +succeeds. Writes may continue into the live tree and are captured by the next +snapshot. Failed publication never advances the tree's durable snapshot or WAL +retention frontier. + Recovery normally opens the newest epoch-valid, fully verified manifest. It may fall back only to the preceding complete retained generation. A caller holding topology recovery evidence may instead request one exact immutable diff --git a/doc/design/tree/design-crowdb-tree-storage.md b/doc/design/tree/design-crowdb-tree-storage.md index d0c3b52a8..e5cbc9058 100644 --- a/doc/design/tree/design-crowdb-tree-storage.md +++ b/doc/design/tree/design-crowdb-tree-storage.md @@ -439,6 +439,10 @@ alike. This is the "4 GiB / 8 GiB btree cache" knob (`capacity_bytes` default metadata under the pool mutex. The page table (`pid -> frame index`) is an open-addressing array, not `std::unordered_map`, to keep the hot path branch-light and cache-friendly. +- **Unused capacity.** Arena allocation preserves the configured frame count + without initializing every unused frame. Fresh and anonymous frames are + zeroed when acquired. A read miss loads and validates the complete frame + before publishing it, preserving initialization and stable frame addresses. - **Pinning and eviction.** A reader pins the leaf/inner frames along its descent, reads via the view, then releases. Pinned frames are never evicted. Eviction uses a **CLOCK (second-chance)** sweep: skip pinned and diff --git a/doc/dev/physical-node-test-plan.md b/doc/dev/physical-node-test-plan.md new file mode 100644 index 000000000..b94b8bcf8 --- /dev/null +++ b/doc/dev/physical-node-test-plan.md @@ -0,0 +1,720 @@ + + + +# Physical Node Test Plan + +This reusable catalog follows category, test case, and scenario. It does not +track the current run's progress. Keep selected scenarios, actual commands, +source/image identities, and results in a separate run directory. Historical +passes do not establish coverage for a new image. + +Upstream: [deployment and recovery](../design/deploy/design-crowdb-deploy.md), +[OCI builds and runtime](../../container/oci-image/README.md), +[container acceptance](../../e2e-test/README.md), and +[KV recovery](../design/kv/design-crowdb-kv-slot.md). + +## 1. Execution contract + +### Environment and scope + +- Use two physical failure domains by default: A `192.168.31.129` and + B `192.168.31.184`. Recheck host identities, interfaces, and addresses each + run; an IP address does not prove resource ownership. +- Use five fixed voters, A1/A2/A3 and B1/B2, with separate persistent volumes + and discovery identities. Port offsets are 0/1/2 on A and 0/1 on B; verify + the values actually accepted by the current launch interface. +- Fix each container at host networking, 1 CPU, 2048 MiB, and 512 PIDs, with + one Tokio worker per service. Record actual cgroup limits, storage capacity, + reserves, and deviations. +- Simulate a host fault only by stopping this run's designated containers on + that host. Do not change shared networking, routes, firewalls, the kernel, + or the Docker daemon. A three-node variant requires a separate profile and + cannot substitute for five-node coverage. +- Use the existing SSH key and noninteractive BatchMode. Supply credentials + privately; exclude passwords, private keys, tokens, login identities, and + commands containing authentication arguments from committed docs/reports. +- Run all builds, tests, and executables, including remote commands, through + `pixi run`. + +### Case selection and results + +- Retain P00-P19 as parent case IDs; `.1`, `.2`, etc. identify independently + verifiable scenarios. The default main sequence covers every parent and its + ordinary scenarios. Scenarios explicitly marked as extensions are required + only by the selected profile; an unselected extension has not passed. +- Freeze scope, concurrency, resources, data size, timeouts, and acceptance + conditions before starting. Record added regressions separately. Do not + remove scenarios, weaken assertions, or change profiles after failure to + obtain an all-pass result. +- For each scenario, record prerequisites, fault target/time, actual actions, + assertions, duration, result, evidence paths, and input identities. Use + `PASS / FAIL / NOT RUN`; an injection that never occurred is a coverage gap. +- Follow dependency order and stop dependent work at the first failure. + Preserve failures and valid passes. After repair, repeat only scenarios + whose inputs or behavior changed, then resume unfinished work. Do not + blindly replay partially executed stages. +- Persist the operation, key/ID, and expected content before each mutation; + persist its acknowledgment after success. Record timeouts/disconnections as + `OutcomeUnknown` and reconcile them read-only. Controllers must not replay + uncertain writes. +- Give fault injection and later read-only audits distinct command-log names. + Preflight those names before mutation. A corrected fresh attempt needs its + own artifact namespace and operation receipt; retain the failed attempt and + establish its mutation outcomes before submitting another operation. +- Do not increase timeouts merely to hide failures, weaken checks, or add caller + retries. An agreed production probe-budget change needs a separate attempt + and preserves earlier failures. P14 observes + the production client's existing explicit safe-retry policy unchanged. +- Complete builds before their consumers and serialize builders sharing an + output directory. Freeze sources before building OCI; retain the source + tree, commit/dirty diff, client digests, and image digest. Reassess affected + coverage after source changes. Separate compilation/linking from acceptance + workloads on shared physical hosts. A contention scenario must declare its + competing workload in advance and collect host pressure, CPU, and disk data. +- Record CI and physical results separately. When the user stops CI inspection, + do not query or infer new runs or make CI completion a physical prerequisite. + Required local gates still apply. + +### Assertions shared by recovery cases + +- Before injection, capture container IDs, volume names/CreatedAt, discovery + UUIDs, node IDs, bootstrap bindings, all 35 services' container/PID/start-time + identities, catalog state, and replica state. +- Observe continuously from injection through recovery and the full data audit. + Complete samples must be no more than five seconds apart. A fault target + missing from `/proc` remains an absent observation; retain sibling samples. +- Permit only the process changes specified by the case. Surviving monitors + must keep publishing fresh, truthful observations during recovery. An offline + target may be unobservable; old Ready evidence is not current evidence. + Freshness and health are separate assertions: updating an old healthy + timestamp cannot conceal Unavailable. +- Recovery gates require current, healthy monitor observations from every + node, newer than its container start and no more than three seconds old at + receipt. Fence all 35 process identities until the full audit ends; do not + reset the baseline to tolerate another restart. Establish the first strict + health check and identity fence within the original service-recovery budget. + An early Ready response cannot defer actual recovery verification beyond + that budget. Independently check strict health again when the full audit + starts. Later health failures or identity changes are audit failures, not + another recovery-wait interval. +- Measure catalog takeover, service recovery, and each of Groups 0/1/3's fixed + checkpoint catch-up separately, each with its original 180-second budget. + Record each gate's own start and duration. Preserve P10's 60-second limit, + P13's 30-second limit, and shorter production-client limits separately. +- After recovery, verify P03 baselines, P04/P05 data, P07 acknowledged live and + deleted records, every historical witness, and this case's new writes. + Read and validate complete content through all five recovered endpoints. +- Actual EC fragment placement determines payload availability; KV quorum + cannot substitute for it. Report temporary availability beyond the fragment + loss budget separately from complete historical correctness after recovery. + +## 2. Category A: hosts, images, and cluster formation + +### P00 Environment, authentication, and images + +- **P00.1 Hosts and resources**: verify both physical hosts, SSH keys, clocks, + interfaces, cross-host listener reachability, Docker/cgroups, free disk, and + memory. Inventory existing resource ownership and record five-node limits + and storage budgets. Fail inadequate preparation without shrinking data. + Record the actual link medium, selected route/interface, and each Wi-Fi + adapter's power-management state. Declare the network profile before loaded + acceptance; neither enabled nor disabled power saving is a universal pass + requirement. Any setting change requires user authorization, before/after + verification on both hosts, and a restoration or continuing-profile decision. + Where available, record kernel ACK policy, CPU idle/governor state, scheduler + pressure, and Wi-Fi power saving as possible timing variables. Their presence + does not prove causation; do not tune host settings automatically. + Do not disconnect an interface or alter shared routing as a test fault. +- **P00.2 Identical image content**: import the same verified image transfer + file on both hosts. Check its digest, manifest-referenced config content hash, + platform, source/runtime labels, and ordered rootfs layers. Docker may expose + a manifest ID on one host and a config ID on the other; comparing ID strings + alone is insufficient. +- **P00.3 Invalid inputs**: offline regressions reject damaged transfers, + source/runtime label mismatches, reordered layers, and architecture mismatch + before any node upgrade starts. +- **Evidence**: redacted host inventory, image/build receipts, actual inspect + results from both hosts, and provenance mapping. + +### P01 Five-node discovery, initialization, and identity + +- **P01.1 Formation**: form five voters through production policy, discovery, + key admission, and storage preparation. Verify five distinct discovery + UUIDs/node IDs, accepted initial_voters, fixed membership, and real listeners. + Use the supported five-voter path without inventing CLI arguments. +- **P01.2 Addresses and ports**: verify each offset is applied once, same-host + ports do not collide, cross-host addresses avoid unreachable loopback, and + both rack/host assignments are correct. +- **P01.3 Identity retention**: restart existing nodes with their original + bindings, volumes, and identities without bootstrapping again. Reject stale + bindings, foreign volumes, and replacement containers reusing an old name. +- **Evidence**: accepted policy, membership/listeners, volume mappings, + bootstrap bindings, and discovery identities. + +### P02 Cluster readiness and continuous observation + +- **P02.1 Baseline**: all five Web endpoints are Ready, with seven services + per node and 35 actual live processes. Verify health, dependencies, and + continued renewal of DiskDB's current ownership lease. +- **P02.2 Freshness**: record receipt times and reject monitor snapshots older + than container start or three seconds. Distinguish process liveness, listener + liveness, and data-service readiness. Cross-host sampling may reuse an + authenticated SSH connection, but each response requires a complete fresh + host sample. Record transport/process-start cost separately from node sampling + time. Reuse cannot relax the five-second complete-observation bound, replay + samples, renew old timestamps, or hide sampling failures. + For loaded cross-host latency, retain idle/load/idle observations with the + same native or protocol audit, source/image, data size, logical deadlines, + concurrency, resource limits, and pinned process identities. Record host I/O, + competing builds, cached-health responses and failed generations separately. + If authorized, compare declared Wi-Fi power-management profiles as distinct + attempts; retain both states and results without rewriting the first failure + or treating a faster comparison as complete recovery acceptance. Use matched + request/response packets and conservative cross-host clock bounds to locate + delay. Record capture scope, startup boundary, kernel drops, and gaps: dropped + or missing capture packets cannot prove network loss. Reconcile collector + failures offline when retained bytes suffice, preserving the original receipt + and qualifying derived evidence separately. + Positive matched packet timestamps include host stack and queue delays; + they are not pure Wi-Fi propagation or server-handler duration. State clock + assumptions and retransmission limits. With capture gaps, an observed request + or reply need not be the earliest emission or arrival. +- **P02.3 Authority degradation**: during Group0 failover/unavailability or a + slow remote observer, the local monitor still reports current processes and + truthful health. Service-intent reads must not block health publication or + diagnostic control. Readiness HTTP requests share independent background + evidence rather than rereading full authority on every poll. Component + regressions cover reads lasting over one second, concurrent readers, expired + or failed evidence, and recovery without page traffic. A warm ChunkDB point + probe must still read the real owner/storage when its client's ordinary + registry refresh becomes due; it must not expand into a full authority scan. + Cover cold discovery, stale-owner rejection, failed-route refresh, and actual + storage errors with the original deadline. When a bulk response + blocks a normal TCP connection, the same storage client's physical probe + reads the original disk's 4096-byte header through the bounded priority lane. + Rejection, short reads, and probe timeout still fail. Compare the same client, + target, and deadline; distinguish response counts from failed probe generations. + Start physical headers as soon as hardware discovery succeeds while the + independent ChunkDB read continues. Exercise the production orchestration + with hardware taking 50ms, ChunkDB 700ms, and a header 100ms: all checks must + complete within the original shared 750ms budget. Held or failed dependencies, + empty/offline hardware, short headers, and cancellation must still fail without + leaked work. With 17 disks, preserve the eight-read batch bound. + This 750ms case remains a component scheduling control. Physical Access + qualification uses the declared two-second production dependency budget and + 2100ms outer watchdog. Record every failed generation and continue sampling; + a successful new generation recovers automatically. Raise every probe failure + below three seconds with its configured timeout, actual duration, error and + recovery; decide jointly before increasing the production budget again. + During a sustained dependency outage, verify continuous negative generations + within the original fault window. Require at least three distinct advancing + generations per fixed endpoint, reject missing/error/Ready samples, and retain + the complete history. Negative evidence may truthfully expire between slow + rounds; do not require every response to be current or renew an old timestamp. + Validate its declared three-second age and require current Ready evidence only + after actual recovery. Preserve process identity and durable-data checks. + Separate deadline correctness from availability qualification: a real read + completing after its declared shared budget requires truthful Unavailable/503, + rather than renewed Ready evidence. That response still fails any applicable + recovered-health availability check; it does not by itself prove a software + defect. Distinguish measured OS/link timing from a source defect established + by a causal regression. Keep formal recovery, identity, data, and availability + assertions unchanged while the cause remains unresolved. + Hold a KV scan frame after its first control byte while the same ChunkDB KV + client issues a real point read. Verify point progress with one additional + connection per endpoint and the existing worker/completion budgets. Cover + rejection, deadline expiry, failed-generation reconnect, explicit pool clear, + and the unchanged ordinary transport as a control. + Under mixed system/user-data metadata, record background KV pages and bytes. + Grow only the other maintenance domain and verify that scoped chunk and + reservation scans do not transfer its pages. Cover all supported purposes, + zero-valued ID suffixes, continuation, foreign slots, and multiple destinations; + unscoped inventory must still return both domains. + Hold one instance's process-lease CAS while another instance remains ready. + A grant pass must attempt each independently ready instance once, atomically + publish the healthy instance's lease/grant pair, and report the failed instance + after the pass. Cover a valid exhausted sequence as another independent + publication failure. Preserve the failed instance's bytes and all process, + assignment, revision, expiry, and replacement fences without a conditional retry. + An error's phase label alone does not identify where a shared budget was + consumed. Never renew an old healthy value or restart a live service solely + for transient unready status. Cover a new process that remains unready and + several listeners exhausting their probe deadlines: background recovery must + yield between passes, and real health probes run concurrently so their waits + do not accumulate and stall the entire snapshot. +- **P02.4 Observer negatives**: reject old snapshots, wrong container/PID/start + identities, OOM, missing siblings, complete-sample gaps over five seconds, + and a second identity change after recovery. +- **P02.5 WAL I/O and executor progress**: use the production multi-thread + runtime with one worker and a real file-backed BlockDevice WAL. Hold an + actual write or durable-sync boundary and verify that independently spawned + apply/RPC-like work still progresses while the append remains unacknowledged. + Release the boundary, then verify write order, durable reopen, and original + error behavior. Keep a current-thread compatibility control; do not claim + that synchronous fallback provides task isolation. In physical slow windows, + correlate Group0 read barrier, apply-fence wait, actual engine apply, WAL + append/sync, and cgroup counters. Record that a shared readiness deadline can + expire before DiskIO starts; a phase label is not proof of slow physical I/O. +- **P02.6 Process observation validity**: follow the + [Console service lifecycle contract](../design/console/design-crowdb-console-ui.md#13-service-lifecycle). + Pair actual owned-process samples with focused backend/UI controls for failed + reads, malformed or incomplete inventories, expired evidence, and wrong Node, + bootstrap/operation epoch or process identity. Missing PID evidence is unknown, + not confirmed absence; retain registered services and successful siblings. + Only a complete, fresh, identity-matched observation can confirm that no process + exists, and an accepted startup must remain pending. Endpoint health cannot + supply a missing PID or prove process absence. +- **P02.7 Read-only plan recovery**: retain a failed-only saved plan and verify + observation continues every two seconds even without Capacity data or another + queued deployment. Unknown evidence keeps waiting or retains the recorded + failure; a current owned running process reconciles progress to deployed. + Include the same PID before and after an observation gap, interrupted-plan + reload, and stale/foreign/expired evidence that must not clear a failure. + Keep the original three-second UI reconciliation assertion. The plan must + issue no automatic Start, Restart or Deploy; explicit Retry alone initiates + another failed lifecycle mutation. Progress recovery does not establish health. +- **Evidence**: complete raw samples, gate receipts, recovery identity fences, + observer results, persisted plan revisions, and lifecycle request counts. + +## 3. Category B: native data, protocols, and scale + +### P03 Baseline data and allocation through five endpoints + +- **P03.1 KV**: verify five-replica mappings and current leaders/leases for + Groups 0/1/3. Issue single and batch writes once, record acknowledgments, and + read complete keys/values through all five endpoints. +- **P03.2 ChunkDB**: allocate and query the same Chunk ID with the production + native client. Check location, size, and identity; a successful new ID cannot + substitute for the original allocation result. +- **P03.3 S3**: independently generate S3 SF1 data with `crowdb-tpc-loader`. + Retain each object's URI, length, and digest, and fully read all eight tables' + files through five endpoints. Retain a run-unique witness of at least 96 KiB + and verify its complete bytes and hash. +- **P03.4 Iceberg**: independently generate eight-table SF1. Check namespace/ + table UUIDs, snapshots, metadata locations, manifest file sets, file lengths/ + digests, decoding of every row, and independent scans. Verify schema, property + commits, and data files separately; footer reads or ten-row samples cannot + replace full reads. For credential expiry, retain the same FileIO and open + stream across the service grant's actual lifetime, then seek/read the same + immutable object. Record original TTL, identity, and refresh scope; do not + substitute table reloads, clock changes, or longer TTLs. Limited scans must + release the native reader before the input file on success and failure. The + verifier must exit normally; a crash or hang after row validation is FAIL. +- **P03.5 Dataset**: generate inline fields, multipart payloads, and immutable + sample snapshots. Fully read each report and field through five endpoints; + verify the actual report count. +- **P03.6 Protocol isolation**: native stress keys must occupy proven safe + ranges outside S3/Iceberg/Dataset metadata prefixes. Adding a test suffix to + a protocol key can still corrupt protocol scans. +- **Evidence**: data/ack ledgers, loader provenance and commands, all generated + reports, and independent verification from five endpoints. + +### P04 A 512-record reproducer and five-owner coverage + +- **P04.1 Historical keys**: run the original 512-record reproducer separately + and record its actual owner coverage. Do not automatically count five owners. +- **P04.2 Formal workload**: read the full catalog and choose run-unique safe + keys within five actual Serving owners' ranges. Use 32 batches of 16 records, + each 64 KiB: 512 records/32 MiB total, retaining the default five-second batch + deadline. Uniform first-byte distribution does not prove routed ownership. +- **P04.3 Integrity**: record each batch's acknowledgments and unknown outcomes, + then fully read it back. Retain split/cutover state, phase latency, and errors. + Reconcile unknown writes read-only without replaying them. + +### P05 Larger working sets and persistent-storage boundaries + +- **P05.1 Scale**: after P04 passes, use a new namespace with at least 4096 + deterministic 64-KiB records, totaling 256 MiB across five actual Serving + owners. Budget mirrors, metadata, WAL, reclamation, and artifacts; keep test + media at least 30% free. +- **P05.2 Extent boundaries**: construct persistent data across file ends using + actual block sizes. After cold open, trigger asynchronous reads crossing one + and multiple extents. Verify full content and exactly one completion. Record + actual offsets, lengths, and boundaries; a large working set alone does not + prove this path ran. +- **P05.3 Error propagation**: component regressions cover missing subsequent + extents, truncated tails, and range overflow. Preserve real I/O errors; reject + short-read success, duplicate completion, or retries masking errors. +- **Evidence**: owners/keys/bytes, full digests, actual extent paths, and + separately stated component and physical coverage. + +## 4. Category C: 30-minute mixed workload and final consistency + +### P06 Sustained mixed workload + +- **P06.1 Parameters**: declare concurrency, operation mix, pacing, key/version + rules, and bounded data volume before starting. Cover both hosts and five + owners; reused or reclaimed data must still produce successful operations. +- **P06.2 Concurrent native and protocol work**: mix native insert, overwrite, + delete, and readback with S3 writes/reads/deletes, Iceberg commit/load/cleanup, + and Dataset publication/readback. +- **P06.3 Effective duration**: sustain at least 30 continuous minutes of + successful overlapping mixed work, excluding builds, deployment, and idle + waits. Waiting after a fixed seed is not load. A repair or controller restart + that interrupts the workload requires a fresh continuous 30-minute run. + +### P07 Acknowledgment semantics and data audit + +- **P07.1 Ledger**: retain operation, key/version, expected length/hash, ack + status, and time before/after mutations. Serialize same-key overwrites or use + an explicit version contract to avoid verifier races. +- **P07.2 Online checks and final full audit**: continuously read acknowledged + data. At completion verify every live key, overwritten value, full protocol + payload, and witness, and continued absence of all acknowledged deletions. +- **P07.3 Unknown outcomes**: retain OutcomeUnknown for timeouts/disconnections. + If present after recovery, verify expected content. Do not assume absence or + replay the mutation to eliminate uncertainty. +- Request failure during normal load, content mismatch, OOM, or an unplanned + process replacement fails that workload stage. + +### P08 Performance and resource observations + +- **P08.1 Complete sampling**: at least every five seconds record throughput, + success/failure/unknown counts, p50/p95/p99/max, CPU, RSS/cgroup memory, free + space, process identities, catalog state, and reclamation. +- **P08.2 Phase statistics**: separate build, startup, seed, sustained load, + fault, and audit durations. Retain successful operations/bytes, effective + overlap, and phase latencies; container liveness alone is insufficient. + Comparisons of sampling overhead or repairs must hold workload, leader host, + routed owner, client/image, and actual cross-host path fixed, recording the + observer's enablement and cost. Leader movement or early workload completion + makes samples incomparable; retain them without claiming improvement. +- **P08.3 Host anomalies**: retain diskstats, I/O/memory pressure, cgroup + throttling/OOM, and readable kernel errors. Low CPU or ample memory cannot + rule out fsync or device I/O stalls. + +## 5. Category D: process, container, and host faults + +### P09 Cold start and image upgrade on original volumes + +- **P09.1 Cold recreation**: stop and recreate all five containers with original + volumes, policy, UUIDs/node IDs, and bootstrap. Verify 35 services, every fixed + checkpoint, and complete historical data through five endpoints. If this also + accepts a new image's startup, explicitly share that actual cold recovery + and its complete observation. Do not inject the same passed scenario again; + retain interrupted startup attempts as failures. +- **P09.2 Same-volume upgrade**: after a production repair, verify the new + image's content identity before replacing containers. Preserve all volume + names/CreatedAt and data identities. Image transfer tar files distribute + software; they are not data-volume backups. +- **P09.3 Operation races**: before recreation recheck old container IDs, + stopped states/times, mounts, and ownership. Stop if another operation started + or replaced a container during preparation; do not blindly remove/create it. + +### P10 One unresponsive ChunkKV + +Combine full catalog and current placement observations at the recovery gate. +All entries may be Serving while a handoff source is preparing or quiescing; +that alone cannot establish recovery. Reject active handoffs and selected moves. +Catalog generation must match before and after health sampling. Distinguish +historical, committed, serviceable overlays from active owner cutover. Retain +the original 180-second recovery budget and the subsequent strict health, +PID/start-time, and complete-audit assertions. + +- **P10.1 Injection**: SIGSTOP only one ChunkKV whose container/PID/start-time + was verified. Do not pause the entire container or a same-named service. +- **P10.2 Recovery**: the old process exits and is replaced by a new PID within + 60 seconds; the other 34 identities remain fixed. Verify old data and this + case's new acknowledged writes. Reject another silent replacement during + the full audit. Track takeover-triggered source checkpoints and return + migration so a recovered process cannot conceal permanently stuck preparation. + Component injection covers synchronous/asynchronous root-publication failure, + sustained unavailability, and intervening writes: complete the pending root's + original barrier before the next checkpoint writes data. Failure remains + Unavailable, the old root stays recoverable, and the recovered new root + contains acknowledged writes; repeated Internal errors are unacceptable. + A terminal-journal component case must drive an actual ChunkKV metadata + mutation during the DiskIO outage. Include a metadata-only operation, such + as bucket creation: object payload writes can fail before metadata publication + and therefore never exercise the intended journal failure or replay. Keep + the original restart, retained-data, and healthy Access identity assertions. +- **P10.3 Failure cleanup**: after retaining errors, resume any surviving + stopped process so injection does not contaminate later work. + +### P11 Cross-host KV lifecycle + +- **P11.1 Follower**: choose a follower on a different physical host from the + current leader. Stop its local KV through its Web endpoint; verify the + completed Stop receipt and actual exit. Remote authority still manages the + node. Restart once, require a new PID and 34 unchanged siblings, and complete + the historical audit. +- **P11.2 Leader**: identify the current leader again and Stop it. Its original + completed receipt remains readable after exit. Wait for surviving nodes' new + authority, then restart the old node once and complete the audit. +- **P11.3 Failover window**: throughout the case, check surviving monitors' + freshness and actual process identities. Final Ready alone can miss blocking + intent reads, old health snapshots, or sibling restarts. Component regressions + cover a reachable first management seed still returning the old leader: + connection failure or rejection without a hint must continue discovery of + another member's new route within existing retry and lease-safety budgets. +- **P11.4 Exit evidence**: before diagnosing native abort, self-join, WAL drain + failure, or another abnormal exit, retain application, RPC, tree, stderr, + and crash/core evidence for that exact PID and container generation. + Save stderr before injection and compare its exact retained prefix afterward; + older panic bytes must not be attributed to the new exit. Successful native + RPC joins do not establish that Tokio tasks or blocking native jobs finished. + A focused component case holds a real service's background sampling read with + a registered timer, drains the service, then drops its runtime. Drain must + retain ownership until the read and its native payload finish; closed sampling + admission must not launch another job. Capture the isolated process's complete + output under an external watchdog. A matching component panic proves that + path is reachable, not the originating future of a physical exit without a + backtrace. Keep existing supervisor grace and I/O deadlines unchanged. +- **P11.5 Leader tenure notifications**: a component regression accepts a + fenced Admin step-down before the old leader loop consumes its notification, + then drives a real election into a new term. The retained notification must + not step down the new leader; a fresh matching request must still fence it + immediately. During physical failover, retain term, role, and step-down + reasons so an unexpected extra election is not dismissed as a slow network. + +### P12 Both B-host nodes offline + +- **P12.1 Actual takeover**: prove B1/B2 are current Serving owners before + stopping them. A's three voters take over the catalog within 180 seconds; + issue one KV write/read and record its actual new acknowledgment. If earlier + faults changed placement, let normal balance restore both owner prerequisites. + Completed independently recoverable transfers must not lose balance eligibility + forever because of historical transition markers. Fail missing prerequisites; + stopping empty owners or changing balance policy cannot manufacture coverage. + Regress the ordering where a lease-excluded target's completed marker was + released before its first checkpoint: the exact current Serving assignment + may still claim root publication for its new epoch. Different owner, epoch, + tree, stream, or active transition must not bypass checks through marker + absence; the old root writer remains fenced. +- **P12.2 New payload while offline**: keep B offline and write a new S3 witness + and Iceberg property commit through A. The new payload's actual placement must + allow A to read it independently; fully read it through all three A endpoints. +- **P12.3 EC boundary**: record old objects' fragments per host and loss budget. + For EC 8+4 with six fragments on B, reconstruction while B is offline cannot + be required or claimed as existing rack protection. Report new-write + availability separately from temporary unavailability of old EC objects. +- **P12.4 Recovery and persistence**: restore B, then recreate B1 and B2 + separately on original volumes. Verify original identities, fixed checkpoints + on five replicas, and all historical data through five endpoints, including + old EC objects unavailable during the outage. +- **Extension**: to test whole-host tolerance for old EC data, separately select + a real placement within its fragment loss budget and fully read old data + during the outage. Three surviving voters cannot establish that guarantee. + +### P13 All three A-host nodes offline + +- **P13.1 Quorum loss**: stop A1/A2/A3. B reports current unavailability within + 30 seconds, and new mutations must not acknowledge success. Retain failed and + unknown outcomes; connection loss does not establish that no write occurred. +- **P13.2 Lease-safety exit**: a surviving ChunkKV exiting after its original + lease expires requires same-process evidence of lease invalidation, deliberate + stop, RPC worker join, and exit. Native abort or unexplained replacement + cannot be classified as normal lease fencing. + New runtime-shutdown panics remain failures even when recovery, data reads, + and native RPC joins pass. Retain actual exit status and backtrace when + available; a later replacement PID cannot supply either retrospectively. +- **P13.3 Recovery and protocol reads**: restore A, verify every service and + fixed checkpoint on five replicas, then audit all native/protocol history. + Independently retain each Iceberg reader's endpoint, PID, table/object/stage, + times, stdout, and exit code before aggregating results. +- **P13.4 Error attribution**: separate the first curl/GetObject error from + interrupted readers and undispatched remainders. Passing raw S3 files cannot + substitute for full Iceberg reads; concurrent output order cannot identify + an endpoint. + +## 6. Category E: allocation, backlog recovery, and space correctness + +### P14 Explicit safe retry at allocation level + +- **P14.1 Bounded fault**: preserve all KV processes and Group0 leader/term/ + lease. Inject Group1 leadership step-down; a current leader must accept at + least one injection within three seconds. +- **P14.2 Actual trigger**: start the production allocation client once and + observe explicit allocation-level Unavailable safe retry. Final success, + transport timeout, or an injection that never occurred is insufficient. +- **P14.3 Existing policy**: release injection and complete allocate/query for + the same ID within the original five-retry/ten-second limit. Retain actual + retry/ack evidence; instrumentation must not change branches or policy. + +### P15 Backlog beyond 1024 slots and full cold recovery + +- **P15.1 Actual gap**: stop a real KV follower and acknowledge at least 1100 + single writes through surviving replicas. Record its offline checkpoint upper + bound and targets before/after writing to prove an actual gap over 1024. +- **P15.2 Catch-up and cold start**: restore the follower and verify bounded + FetchGap concurrency and actual progress. Then stop all five nodes and cold + start on original volumes. Freeze Groups 0/1/3's acknowledgment targets before + the fault; all five healthy replica engines per group must reach them before + full data checks. Do not move targets with subsequent writes. +- **Extension P15.3**: separately create physical reclaimed-WAL, empty-sealed-WAL, + and high-promise/old-chosen boundaries and retain trigger evidence. An absent + trigger remains a coverage gap; component regressions or a normal large-gap + pass do not establish these physical boundaries. + +### P16 Final data and DiskDB occupancy + +- **P16.1 Final gate**: all catalog entries are Serving, five endpoints are + Ready, and 35 services are healthy. Require the same identities in the next + sample and at the end of the final audit. +- **P16.2 New and old data**: verify a new KV batch, new ChunkDB allocate/query, + and all historical data separately. Preserve distinctions among + acknowledgments, acknowledged deletions, and OutcomeUnknown. +- **P16.3 Full occupancy**: scan all this run's DiskDB occupancy, + covering every disk, zone, and continuation without truncation. Verify live + mirror segments do not overlap across chunks; retain actual live, allocated, + free, and reserved counts. +- **P16.4 Consistent inventory**: a bounded current-version scan is not an MVCC + snapshot. Record one initial cutoff for complete busy-allocation and + `/chunk/` scans. Either prove both prefixes unchanged through a valid + completion fence, or reconstruct both inventories at that fence by replaying + every intervening journal put and delete. Do not ignore metadata fields or + discard changed records. Apply all scope, completeness, allocation, bounds + and non-overlap checks to the resulting inventory. +- **P16.5 Completion fence and journal**: a coalesced ReadIndex may return a + conservative earlier read point. For reconstruction, submit one isolated + test-control marker to the same Group1 after both scans, outside the audited + prefixes. Record intent before submission, use a unique key, and require a + positive acknowledgment with a matching transport correlation ID and a + chosen revision beyond the initial cutoff. Record the logical request ID + separately. An unknown outcome fails without automatic resubmission. All + scans and replay are read-only; the marker is a separate control write. + Fetch the complete bounded journal interval for both prefixes without + splitting operations sharing a slot. Reject GC gaps, missing history, + truncation, a read slot below the fence, and foreign or unordered operations. + Retain raw scans, operations, reconstructed maps, and marker outcome. +- **P16.6 Failed stability evidence**: matching busy-allocation or `/chunk/` + mutations must fail an unchanged-inventory guard. Capture the exact failed + prefix and slot interval immediately, before journal GC or expensive failure + collection, with one bounded 20-second read-only diagnostic. Retain raw and + decoded keys, values, operations and source identities. Diagnostic failure + preserves the original audit exception. A GC gap or truncation leaves the + changed fields unknown. A later consistent audit is a separate qualification, + not proof about the failed scan or permission to weaken its assertions. + +## 7. Category F: first-failure diagnosis, repair, and cleanup + +### P17 Failure evidence and classification + +- **P17.1 First error**: print the failing case, operation, and complete error; + enable `CROWDB_E2E_PRINT_SERVICE_ERRORS=1` during diagnosis. Retain original + client output rather than only a wrapper panic. +- **P17.2 Shared timeline**: retain inspect/exit/OOM, current PID/start-time, + health timestamps, KV role/term/lease/checkpoint, catalog/transfer, DiskIO, + and disk resources. For a cross-host timeout, distinguish connection setup, + request transmission, and response headers/body. Match the connection to the + current service's socket ownership. A generic client timeout alone does not + identify its phase. A scoped two-host packet capture must retain kernel + timestamps, clock bounds, snapshot/size limits, and capture drop counts; + capture drops alone do not establish network loss. Keep diagnostic helpers + outside the application's constrained cgroups, record their interference, + and retire only their exact owned identities. Reusing an observer connection + must still fetch a fresh response for every scheduled sample. Report a failed + exchange before a later scheduled reconnect; do not replay a cached body, + retry inside the sample, or extend the request timeout. +- **P17.3 Native and host evidence**: analyze crash/core before changing code + for an abnormal exit. For I/O stalls, add diskstats/pressure and accessible + blocked-thread stacks. Separate observations, inferences, and unproven causes. + A later pass does not prove an old abort without a stack or a timeout without + an endpoint label had the same cause. +- **P17.4 Complete logs**: select application, RPC, metrics, and tree logs by + actual PID, container start, and log type. Historical stderr is not a new + process error, but a nonempty old log must not hide new errors. Retain before/ + after evidence or a delta bound to this execution. + +### P18 Repair and continuation + +- **P18.1 Minimal reproduction**: find the first divergence and classify code, + ordering/injection, observer, or environment causes. Add a focused regression + that fails on old code, then run the affected group and relevant gates. +- **P18.2 Actual inputs**: rebuild, verify, and distribute images/clients after + production inputs change. Repair dependencies that make later cases slow, + hang, or fail before dispatching more work against them. +- **P18.3 Continuation ledger**: give each attempt a separate identity and + non-overwritable receipt. Retain original failures, acknowledged writes, and + unknown outcomes. Resetting baselines or moving stop markers cannot erase + observer failures. Record repairs/environment checks and new audit scope + before resuming unfinished work. Before startup or other mutations, verify + each stop/investigation marker's origin, time, and failure. Retain an old + marker and continue only after proving its failure repaired and current + identities unchanged. Unknown, new, or unrepaired markers keep blocking work. +- **P18.4 Coverage boundaries**: distinguish component, container, physical, + and soak coverage. A focused pass on a new image does not inherit all old + coverage. An interrupted workload needs a new 30-minute run. Explain whether + later recovery changes affect old soak inputs without repeating unaffected + work merely to produce a same-image pass. Sharing a completed scenario + requires matching source/epoch/image, actual fault receipts, complete + observations, and recovered identity fences. Diagnostic sampling gaps are + not continuous observation; a complete fault/audit interval requires a fresh + complete scenario. Keep an unchanged-image revalidation separate from a + causal repair: preserve the first failed attempt and carry an unexplained + failure into the final report. A passing revalidation does not explain that + failure or establish a long-term stability rate. + +### P19 Stop, retain evidence, and retire exact resources + +- **P19.1 Both-host preflight**: freeze the complete deletion inventory for + both hosts. Verify run labels, names, container IDs, original volume names/ + CreatedAt, mounts, and final stopped states/times. Stop on any change; a + matching name alone does not authorize deletion. +- **P19.2 Evidence after stop**: after stopping this run's nodes, retain complete + service logs, crashes, inspect results, and data ledgers. Finish evidence + capture on both hosts before the first deletion. +- **P19.3 Volume backup policy**: default to no additional full-volume archives + and follow this run's user retention/cleanup instructions. Retain existing + private backups. Create and verify extra volume backups only when explicitly + requested. Service logs, image transfers, and volume backups serve different + purposes and cannot substitute for each other. +- **P19.4 Exact deletion**: delete only authorized owned resources that passed + preflight, with counts exactly matching the inventory. Do not use daemon-wide + prune or clean shared runtime/other tasks' resources. +- **P19.5 Post-cleanup verification**: confirm this run's target resources are + absent on both hosts, their listeners are closed, and SSH key access still + works. Retain run artifacts, original errors, and agreed files. Disclose any + earlier out-of-scope deletion or unrecoverable data loss in the report. + +## 8. Run deliverables and acceptance conclusion + +Use `.crowdb-runtime/artifacts/physical-nodes//`. Retain actual harness +sources and commands. No single repository Pixi task currently implements this +entire catalog; do not depend on old temporary files or describe an existing +one-command runner that does not exist. + +- `run.json`: selected parent/child scenarios and extension profiles, hosts, + node/port/resource mappings, ownership inventory, source/image/client + identities, dirty diff, harness provenance, and build/transfer receipts. +- `results.json`: each scenario's PASS/FAIL/NOT RUN, independent gate start/ + limit/duration, actual injection trigger, unrun reason, attempt, and evidence + paths. Never overwrite old results. +- `data-ledger.jsonl`, `metrics.jsonl`, `checkpoints.json`: data and acknowledgment + semantics, phase statistics, fixed targets, and actual recovery progress. + Segment large files. +- `loader/` and `audits/`: protocol source versions, generation/injection/ + verification commands, complete reports, and independent endpoint results. + Mark unimplemented paths NOT RUN; other protocols or components cannot + substitute for them. +- `observations/` and `first-failures/`: continuous injection-through-audit + observations, identity fences, first errors, complete logs, crashes, and + explicitly unproven causes. +- `cleanup.json` and `summary.md`: exact retirement, actual source-generation + coverage, unfinished/untriggered scenarios, retention/loss facts, and next + steps. Distinguish ending a run from passing every selected required scenario. + +Conclusions cover only explicitly selected scenarios actually passed in this +run. Track human joint review separately; a test pass cannot complete it. + +## 9. Implementation and regression entry points + +- Startup/images: `container/oci-image/{run-node,runtime,artifact,transfer}.py`, + `container/single-node-container/entrypoint.sh`, and + `container/crowdb-monitor/src/node/startup/policy.rs`. +- Formation/observation: `lib/crowdb-protocol/src/mgmt/node/`, + `app/crowdb-web/src/node/`, + `lib/crowdb-console-shared/src/ops/cluster/readiness/`, and + `container/crowdb-monitor/src/node/runtime/`. +- Data/recovery: `lib/crowdb-{kv,kv-client,chunk-kv-client,chunkdb-client}/`, + `app/crowdb-chunkdb/src/topology/`, + `lib/crowdb-chunk-stream/src/production_chunk/`, and + `lib/crowdb-tree/src/backend/local/`. +- Components: KV backlog/FetchGap durability, ChunkStream finalized-empty, + Monitor discovery/health/service receipts, ChunkDB allocation retry/topology + availability, and BlockAsyncPageStore reads across extents. Select groups + affected by actual changes and run them through `pixi run`. +- Containers/protocols: `e2e-test/tests/deployment/`, `e2e-test/tests/console/`, + `e2e-test/tests/access_iceberg/`, and corresponding S3/native SDK clients. + Count them separately from real cross-host faults and the continuous + 30-minute workload. diff --git a/doc/doc_index.md b/doc/doc_index.md index 903bc2660..8497de03a 100644 --- a/doc/doc_index.md +++ b/doc/doc_index.md @@ -47,21 +47,23 @@ Temporary plans live under `doc/working/`; flow analyses live under ## Dev Environment (`doc/dev/`) -| Doc | When to read | -| ---------------------------- | ---------------------------------------------------------------------- | -| `doc/dev/crash_debugging.md` | Core collection, host setup, GDB, and exact-build symbols. | -| `doc/dev/env_setup.md` | Benchmark commands, sentinels, prerequisites, and perf-counter setup. | -| `doc/dev/hyper_fork.md` | Hyper fork branches, submodule, build, sync, validation, and recovery. | -| `tools/README.md` | Tool directories, Pixi task entry points, CI checks, and suite timing. | +| Doc | When to read | +| ------------------------------------ | -------------------------------------------------------------------------------- | +| `doc/dev/crash_debugging.md` | Core collection, host setup, GDB, and exact-build symbols. | +| `doc/dev/env_setup.md` | Benchmark commands, sentinels, prerequisites, and perf-counter setup. | +| `doc/dev/hyper_fork.md` | Hyper fork branches, submodule, build, sync, validation, and recovery. | +| `doc/dev/physical-node-test-plan.md` | Physical cases, Wi-Fi profiles, loaded latency, recovery gates and evidence. | +| `tools/README.md` | Tool directories, Pixi task entry points, CI checks, and suite timing. | ## Component Acceptance -| Doc | When to read | -| ------------------------------------------- | ----------------------------------------------------- | -| `app/crowdb-access-server/tests/README.md` | Access gates, SDK commands and Dataset measurements. | -| `e2e-test/README.md` | Container domains, browser/SDK ownership and timings. | -| `container/single-node-container/README.md` | Container build, publication and tested client setup. | -| `e2e-test/clients/ecosystem/README.md` | Manual pinned client/engine matrix and exclusions. | +| Doc | When to read | +| ------------------------------------------- | ------------------------------------------------------- | +| `app/crowdb-access-server/tests/README.md` | Access gates, SDK commands and Dataset measurements. | +| `e2e-test/README.md` | Container domains, browser/SDK ownership and timings. | +| `container/single-node-container/README.md` | Container build, publication and tested client setup. | +| `container/oci-image/README.md` | Shared OCI builds, launch examples and retained inputs. | +| `e2e-test/clients/ecosystem/README.md` | Manual pinned client/engine matrix and exclusions. | ## Project Files (repo root) diff --git a/doc/working/plan-ci-stability-review.md b/doc/working/plan-ci-stability-review.md new file mode 100644 index 000000000..9df6b1bdc --- /dev/null +++ b/doc/working/plan-ci-stability-review.md @@ -0,0 +1,42 @@ + + + +# CI Stability Review Plan + +Run: https://github.com/buzzcrow/crowdb/actions/runs/41816831055 + +All four failed jobs were repaired and locally verified on 2026-11-04: +AccessTests completes every original stage; ContainerKV passes 50 fixtures and +58 client phases with isolation; Console native operations passes 19 owners; +Console native data passes all 14 original owners. Rust fmt, all-target clippy, +314 fixture controls and CI task coverage pass. Original failures remain in the +evidence; this records local verification, not a later remote CI result. + +As requested, this persistent plan retains only unfinished follow-up work. +Completed CI execution and review tasks have been removed. + +CI evidence from repository root: +`.crowdb-runtime/artifacts/ci/run-41816831055/`. + +The Dataset import follow-up in +[run 41829384144](https://github.com/buzzcrow/crowdb/actions/runs/41829384144/job/126583192879) +is repaired and locally verified: preparation and browser execution each retain +360 seconds in the same fixture, instead of sharing one client budget. All five +Dataset owners pass at CI concurrency, including full production placement, +512 records, 12 partitions and three owners. Original browser assertions remain +unchanged. All 316 runner controls, Rust format, clippy and CI coverage pass. +Evidence from repository root: +`.crowdb-runtime/artifacts/ci/job-126583192879/`. + +## Remaining task: deferred historical attribution + +- [ ] **P13 first aggregate WAL release error (deferred)**: identify its exact + historical partition/phase and durable release outcome if further attribution + is required. Saved logs map six failing streams but do not identify the first + aggregate branch. A local server reproduction needs typed partition, stream, + epoch, lifecycle, release phase and first stalled cause. Preserve lease + exclusion and replay. Existing real-worker controls pass and establish + `WriteStalled` before release with zero payload IO after authority revocation; + they do not justify a production fencing change. + Evidence from repository root: + `.crowdb-runtime/artifacts/physical-nodes/physical-20261031T194149Z/repair/readiness-probe-budget-2s/wal-drain-research-20261104/findings.md`. diff --git a/e2e-test/README.md b/e2e-test/README.md index 5aae427c7..3c957ac21 100644 --- a/e2e-test/README.md +++ b/e2e-test/README.md @@ -15,6 +15,12 @@ Test clients run on the host against inspected node-container IPs and fixed ports, with three node containers for a three-node cluster. Fresh one- through eight-node profiles exercise management, membership, async operations and multi-store routing. Host bridge IP reachability is required; remote Docker and Linux-VM routing are not implicitly supported. +The CLI terminal-writer fault also requires host `iptables-nft`, its shared +libraries and xtables extensions. A short-lived release-image helper receives +NET_ADMIN only inside each verified owned CLI bridge namespace and drops the +accepted DiskIO listener for forty seconds. Monitor and logging keep running; +the helper never enters the host network namespace. Tool fingerprints, rule +verification and failure-safe rule cleanup remain in the case artifacts. Containers supply server subsets or clusters from the formal release image; they are not test execution environments. Domain profiles configure that same server artifact rather than introducing separate test-server builds. diff --git a/e2e-test/browser_batch.py b/e2e-test/browser_batch.py index 0fb434663..e972fe00e 100644 --- a/e2e-test/browser_batch.py +++ b/e2e-test/browser_batch.py @@ -4,23 +4,46 @@ import concurrent.futures from contextlib import ExitStack +from fixture_result import CaseNotExecuted from resource_dispatch import resource_batches class BatchFixture: """Retain each client's failure until its own deferred lifecycle teardown.""" - def __init__(self, cluster): + def __init__(self, cluster, case_name, errors): self.cluster = cluster + self.case_name = getattr(cluster, 'case_name', case_name) + self.errors = errors self.failure = None def __enter__(self): return self.cluster.__enter__() def __exit__(self, kind, error, traceback): - if error is None and self.failure is not None: + if self.failure is not None: error = self.failure kind, traceback = type(error), error.__traceback__ - return self.cluster.__exit__(kind, error, traceback) + try: + return self.cluster.__exit__(kind, error, traceback) + except Exception as cleanup_error: + # An unwind error belongs to this fixture, not the next peer. + self.errors.append(cleanup_error) + return False + + +def _enter_fixtures(stack, owners, errors): + clusters = [] + for owner in owners: + try: + clusters.append(stack.enter_context(owner)) + except BaseException as error: + # Keep startup first even when peer cleanup also fails. + if isinstance(error, Exception): + errors.append(error) + for entered in owners[:len(clusters)]: + entered.failure = CaseNotExecuted(owner.case_name) + raise + return clusters def run_browser_batches(cases, slots, prepare, execute, fail_fast=False, @@ -31,8 +54,9 @@ def run_browser_batches(cases, slots, prepare, execute, fail_fast=False, # Keep all fixtures in this bounded batch until every client has ended. try: with ExitStack() as stack: - owners = [BatchFixture(prepare(case)) for case in batch] - clusters = [stack.enter_context(owner) for owner in owners] + owners = [BatchFixture(prepare(case), getattr(case, 'name', str(case)), errors) + for case in batch] + clusters = _enter_fixtures(stack, owners, errors) with concurrent.futures.ThreadPoolExecutor(max_workers=slots) as pool: futures = {pool.submit(execute, case, cluster): owner for case, cluster, owner in zip(batch, clusters, owners)} @@ -43,7 +67,8 @@ def run_browser_batches(cases, slots, prepare, execute, fail_fast=False, futures[future].failure = error errors.append(error) except Exception as error: - errors.append(error) + if not any(error is previous for previous in errors): + errors.append(error) if fail_fast and errors: break return errors diff --git a/e2e-test/clients/console-ui/e2e/fixtures/candidateSetup.ts b/e2e-test/clients/console-ui/e2e/fixtures/candidateSetup.ts index c93ec1036..40ed9dd19 100644 --- a/e2e-test/clients/console-ui/e2e/fixtures/candidateSetup.ts +++ b/e2e-test/clients/console-ui/e2e/fixtures/candidateSetup.ts @@ -18,8 +18,10 @@ export async function reviewDisklessCandidate(dialog: Locator, request?: APIRequ await expect(dialog.getByLabel('Managed file capacity (GiB)', { exact: true })).toHaveCount(0); for (const name of ['crowdb-paxos-kv', 'crowdb-disk-db', 'crowdb-disk-io', 'crowdb-chunk-db', 'crowdb-chunk-kv', 'crowdb-access-server']) { const service = dialog.getByRole('checkbox', { name, exact: true }); - await expect(service).not.toBeChecked(); + if (name === 'crowdb-paxos-kv') await expect(service).toBeChecked(); + else await expect(service).not.toBeChecked(); await expect(service).toBeDisabled(); } + await expect(dialog.getByText("This node's startup settings fix these choices.", { exact: true })).toBeVisible(); await dialog.getByRole('button', { name: 'Join and start', exact: true }).click(); } diff --git a/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts b/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts index 47189ce07..a568ea0df 100644 --- a/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts +++ b/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts @@ -23,7 +23,7 @@ export function observeDefaultDeployments(page: Page) { const identity = key(response); if (identity) replies.set(identity, [...(replies.get(identity) ?? []), response]); }); - const expectedWait = (response: Response, kind: string) => response.status() === 409 && ['chunkdb', 'chunk-kv'].includes(kind); + const expectedWait = (response: Response, kind: string) => response.status() === 409 && ['chunkdb', 'chunk-kv', 'access-server'].includes(kind); async function completed(nodes: number[], kind: string) { const timeout = process.platform === 'darwin' ? 30_000 : 10_000; let found: { node: number; response: Response } | undefined; @@ -65,7 +65,8 @@ export function observeDefaultDeployments(page: Page) { } async function verifyResponse(response: Response, kind: string) { if (expectedWait(response, kind)) { - expect((await response.json()).error).toMatch(/^Waiting: /); + const node = Number(new URL(response.url()).pathname.split('/')[3]); + expect(await response.json()).toMatchObject({ error: expect.stringMatching(/^Waiting: /), service: kind, node_id: node, retryable: true }); } else { expect(response.status(), await response.text()).toBe(201); } diff --git a/e2e-test/clients/console-ui/e2e/fixtures/icebergInspection.py b/e2e-test/clients/console-ui/e2e/fixtures/icebergInspection.py index f2be9973f..91bcfbef4 100644 --- a/e2e-test/clients/console-ui/e2e/fixtures/icebergInspection.py +++ b/e2e-test/clients/console-ui/e2e/fixtures/icebergInspection.py @@ -19,7 +19,7 @@ properties={"write.parquet.row-group-limit": "2"}, ) for first in (0, 44): - table.append(pa.table({"id": list(range(first, first + 44)), "message": ["雪", "alpha", "beta", "gamma"] * 11})) + table.append(pa.table({"id": list(range(first, first + 44)), "message": ["\u96ea", "alpha", "beta", "gamma"] * 11})) print("Committed two snapshots with native Parquet data and Avro manifests") many = catalog.create_table( (namespace, "many_files"), diff --git a/e2e-test/clients/console-ui/e2e/fixtures/icebergRoundTrip.py b/e2e-test/clients/console-ui/e2e/fixtures/icebergRoundTrip.py index bf42aee4a..ee9338121 100644 --- a/e2e-test/clients/console-ui/e2e/fixtures/icebergRoundTrip.py +++ b/e2e-test/clients/console-ui/e2e/fixtures/icebergRoundTrip.py @@ -11,7 +11,7 @@ table = catalog.load_table((sys.argv[2], "events")) # The UI create response vends a writer grant; Console GETs vend read grants. table.io = load_file_io({**table.io.properties, **json.load(sys.stdin)}, location=table.location()) -expected = [{"id": 1, "message": "three nodes"}, {"id": 2, "message": "雪"}, {"id": 3, "message": "S3 + Iceberg"}] +expected = [{"id": 1, "message": "three nodes"}, {"id": 2, "message": "\u96ea"}, {"id": 3, "message": "S3 + Iceberg"}] table.append(pa.Table.from_pylist(expected, schema=pa.schema([("id", pa.int64()), ("message", pa.string())]))) actual = catalog.load_table((sys.argv[2], "events")).scan().to_arrow().to_pylist() assert sorted(actual, key=lambda row: row["id"]) == expected, actual diff --git a/e2e-test/clients/console-ui/e2e/fixtures/journalRuntimeRequest.ts b/e2e-test/clients/console-ui/e2e/fixtures/journalRuntimeRequest.ts new file mode 100644 index 000000000..e693638fc --- /dev/null +++ b/e2e-test/clients/console-ui/e2e/fixtures/journalRuntimeRequest.ts @@ -0,0 +1,52 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +import type { Page, Request, Response } from '@playwright/test'; + +export interface JournalRuntimeTarget { + origin: string; id: string; epoch: string; generation: string; page: number; offset: number; +} + +export function matchesJournalRuntimeRequest(request: Pick, target: JournalRuntimeTarget): boolean { + if (request.method() !== 'GET') return false; + const url = new URL(request.url()); + const query = { id: target.id, epoch: target.epoch, generation: target.generation, + page: String(target.page), offset: String(target.offset), stream_offset: '0' }; + return url.origin === target.origin && url.pathname === '/api/chunk-kv/runtime' + && [...url.searchParams.keys()].length === Object.keys(query).length + && Object.entries(query).every(([name, value]) => url.searchParams.get(name) === value); +} + +/** Observe the GET started by this action, never a late response to an earlier selection. */ +export async function journalRuntimeResponse(page: Pick, target: JournalRuntimeTarget, + action: () => Promise, remaining: () => number): Promise { + const budget = remaining(); + if (budget <= 0) throw new Error('Journal observation exceeded its three-second deadline'); + let owned: Request | null = null; + let receive!: (response: Response) => void; + let fail!: (error: Error) => void; + const observed = new Promise((resolve, reject) => { receive = resolve; fail = reject; }); + const onRequest = (request: Request) => { + if (!owned && matchesJournalRuntimeRequest(request, target)) owned = request; + }; + const onResponse = (response: Response) => { + if (response.request() !== owned) return; + void response.finished().then(error => { if (error) fail(error); else receive(response); }, fail); + }; + const onFailure = (request: Request) => { + if (request === owned) fail(new Error(request.failure()?.errorText ?? 'Journal runtime request failed')); + }; + const timeout = setTimeout(() => fail(new Error(`Journal runtime response timed out for ${target.id}, epoch ${target.epoch}, catalog ${target.generation}`)), budget); + page.on('request', onRequest); + page.on('response', onResponse); + page.on('requestfailed', onFailure); + try { + const [response] = await Promise.all([observed, Promise.resolve().then(action)]); + return response; + } finally { + clearTimeout(timeout); + page.off('request', onRequest); + page.off('response', onResponse); + page.off('requestfailed', onFailure); + } +} diff --git a/e2e-test/clients/console-ui/e2e/fixtures/nativeProcess.ts b/e2e-test/clients/console-ui/e2e/fixtures/nativeProcess.ts index bda727381..8b3e0d3ed 100644 --- a/e2e-test/clients/console-ui/e2e/fixtures/nativeProcess.ts +++ b/e2e-test/clients/console-ui/e2e/fixtures/nativeProcess.ts @@ -7,17 +7,35 @@ import { containerControl } from './containerControl'; const known = new Set(); const observation = (body: object) => containerControl(process.env.CROWDB_NATIVE_OBSERVATION_CONTROL, body); +export interface NativeServiceState { + pid: number | null; + generation: number; + operation_id: string | null; +} + +/** Read typed service identities from this fixture's owned monitor. */ +export async function nativeServices(node: number): Promise> { + const status = await observation({ action: 'observe-native-services', node }); + for (const service of Object.values(status.services) as NativeServiceState[]) { + if (service.pid) known.add(`${node}/${service.pid}`); + } + return status.services; +} + +/** One OS observation after the owning monitor has supplied the PID. */ +export async function observedNativeProcessAlive(node: number, pid: number): Promise { + if (!known.has(`${node}/${pid}`)) throw new Error('Native PID has not been observed in this owned node'); + return (await observation({ action: 'observe-native-process', node, pid })).alive; +} + /** Observe only a PID recorded by this fixture's actual monitor. */ export async function nativeProcessAlive(node: number, pid: number): Promise { const key = `${node}/${pid}`; if (!known.has(key)) { await expect.poll(async () => { - const status = await observation({ action: 'observe-native-services', node }); - for (const service of Object.values(status.services) as { pid?: number }[]) { - if (service.pid) known.add(`${node}/${service.pid}`); - } + await nativeServices(node); return known.has(key); }, { intervals: [100] }).toBe(true); } - return (await observation({ action: 'observe-native-process', node, pid })).alive; + return observedNativeProcessAlive(node, pid); } diff --git a/e2e-test/clients/console-ui/e2e/fixtures/nativeServiceLifecycle.ts b/e2e-test/clients/console-ui/e2e/fixtures/nativeServiceLifecycle.ts new file mode 100644 index 000000000..b5b15d17d --- /dev/null +++ b/e2e-test/clients/console-ui/e2e/fixtures/nativeServiceLifecycle.ts @@ -0,0 +1,154 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +import type { APIRequestContext, Page } from '@playwright/test'; +import { expect } from './realBackend'; +import { nativeProcessAlive, nativeServices, observedNativeProcessAlive } from './nativeProcess'; +import { step } from './stepTimer'; + +interface NativeServer { + id: string; + mgmt_url?: string; + node_id: number; + pid?: number; + pid_node_id?: number; + health: string; + service_type: string; + start_pending: boolean; +} + +export async function nativeServerInventory(request: APIRequestContext): Promise { + const response = await request.get('/api/servers'); + expect(response.ok(), await response.text()).toBe(true); + return response.json(); +} + +/** Observe local dependency convergence before the single Access restart. */ +export async function accessDependencyReady(request: APIRequestContext, node: number) { + const dependency = (await nativeServerInventory(request)).find(row => row.node_id === node && row.service_type === 'chunk-kv'); + expect(dependency).toBeDefined(); + expect(dependency.id).toMatch(/^chunk-kv-\d+$/); + expect(dependency.mgmt_url).toMatch(/^https?:\/\//); + const instance = Number(dependency.id.slice('chunk-kv-'.length)); + const started = performance.now(); + const deadline = started + 10_000; + let observation: { status: number; health: unknown } | null = null; + const diagnostic = setTimeout(() => { + console.warn('[DEPENDENCY] Chunk-KV convergence exceeded 3000ms', JSON.stringify({ node, instance, observation })); + }, 3000); + try { + await step('native lifecycle: Access dependency readiness', () => expect.poll(async () => { + const remaining = Math.ceil(deadline - performance.now()); + if (remaining <= 0) return false; + const response = await request.get(new URL('/ready', dependency.mgmt_url).href, { timeout: remaining }); + const health = await response.json(); + observation = { status: response.status(), health }; + expect([200, 503], await response.text()).toContain(response.status()); + expect(health.instance_id).toBe(instance); + return response.status() === 200 && health.ready === true; + }, { timeout: 10_000, intervals: [100] }).toBe(true)); + } finally { + clearTimeout(diagnostic); + console.log('[DEPENDENCY] Chunk-KV convergence finished', JSON.stringify({ node, instance, elapsed_ms: Math.round(performance.now() - started), observation })); + } +} + +async function siblingIdentities(target: string) { + const nodes = await Promise.all([0, 1, 2].map(node => nativeServices(node))); + return nodes.flatMap((services, node) => Object.entries(services) + .filter(([id]) => node !== 0 || id !== target) + .map(([id, { pid, generation, operation_id }]) => ({ node, id, pid, generation, operation_id }))) + .sort((left, right) => `${left.node}/${left.id}`.localeCompare(`${right.node}/${right.id}`)); +} + +const registration = ({ pid: _pid, pid_node_id: _namespace, health: _health, start_pending: _pending, ...row }: NativeServer) => row; + +async function stopAuxiliary(page: Page, request: APIRequestContext, original: NativeServer, kind: string, label: string, prefix: string) { + const sidebar = page.getByRole('complementary', { name: 'Cluster tree sidebar' }); + await step(`native lifecycle: ${original.id} Stop response`, async () => { + await sidebar.getByRole('button', { name: 'N-1', exact: true }).click({ button: 'right' }); + await page.getByRole('menuitem', { name: `${prefix}-1 Running`, exact: true }).hover(); + const stopped = page.waitForResponse(response => + new URL(response.url()).pathname === `/api/services/${original.id}/stop` + && response.request().method() === 'POST'); + await page.getByRole('menuitem', { name: `Stop ${label}`, exact: true }).click(); + const response = await stopped; + expect(response.status(), await response.text()).toBe(200); + expect(await response.json()).toMatchObject({ id: original.id, pid: null, removed: false, data_preserved: true }); + }); + await expect(page.locator('header').getByRole('button', { name: 'Refresh', exact: true }).locator('svg')).not.toHaveClass(/tw-animate-spin/); + await step(`native lifecycle: ${original.id} stopped process and inventory`, async () => { + await expect.poll(() => nativeProcessAlive(0, original.pid), { intervals: [100] }).toBe(false); + await expect.poll(async () => { + const current = (await nativeServerInventory(request)).find(row => row.id === original.id); + return { id: current?.id, pid: current?.pid ?? null, ...(kind === 'access-server' ? { health: current?.health } : {}) }; + }, { intervals: [100] }).toEqual({ id: original.id, pid: null, ...(kind === 'access-server' ? { health: 'down' } : {}) }); + }); +} + +async function startAuxiliary(page: Page, request: APIRequestContext, original: NativeServer, kind: string, label: string, prefix: string, oldOperation: string) { + const sidebar = page.getByRole('complementary', { name: 'Cluster tree sidebar' }); + if (kind === 'access-server') await accessDependencyReady(request, original.node_id); + let submitted = false; + try { + await sidebar.getByRole('button', { name: 'N-1', exact: true }).click({ button: 'right' }); + await page.getByRole('menuitem', { name: `${prefix}-1 Stopped`, exact: true }).hover(); + // Preserve the existing response budget; the combined process assertion stays 3s. + const started = page.waitForResponse(response => + new URL(response.url()).pathname === `/api/services/${original.id}/restart` + && response.request().method() === 'POST', { timeout: 5_000 }); + const startAt = performance.now(); + submitted = true; + const response = await step(`native lifecycle: ${original.id} Start response`, async () => { + await page.getByRole('menuitem', { name: `Start ${label}`, exact: true }).click(); + return started; + }); + console.log(`[START] ${original.id}: ${(performance.now() - startAt).toFixed(0)}ms`); + expect(response.status(), await response.text()).toBe(200); + const accepted = await response.json(); + expect(accepted).toMatchObject({ id: original.id, removed: false, data_preserved: true }); + expect(Number.isInteger(accepted.pid)).toBe(true); + expect(accepted.pid).toBeGreaterThan(1); + expect(accepted.pid).not.toBe(original.pid); + await step(`native lifecycle: ${original.id} new process, inventory and Access recovery`, () => expect.poll(async () => { + const inventory = await nativeServerInventory(request); + const current = inventory.find(row => row.id === original.id); + const access = inventory.find(row => row.node_id === original.node_id && row.service_type === 'access-server'); + const actual = (await nativeServices(0))[original.id]; + // A new backend PID can precede dependent Access readiness. Close that + // outage inside this same three-second assertion before the next cycle. + return current?.pid === accepted.pid && current.pid_node_id === original.node_id + && actual?.pid === accepted.pid && typeof actual.operation_id === 'string' + && actual.operation_id !== oldOperation && await observedNativeProcessAlive(0, accepted.pid) + && (kind !== 'access-server' || current.health === 'up') && access?.health === 'up'; + }, { intervals: [100] }).toBe(true)); + const current = (await nativeServerInventory(request)).filter(row => row.id === original.id); + expect(current).toHaveLength(1); + expect(registration(current[0])).toEqual(registration(original)); + } finally { + if (!submitted) { + const current = (await nativeServerInventory(request)).find(row => row.id === original.id); + if (!current?.pid) expect((await request.post(`/api/services/${original.id}/restart`, { data: {} })).ok()).toBe(true); + } + } +} + +export async function cycleAuxiliaryService(page: Page, request: APIRequestContext, kind: string, label: string, prefix: string) { + const original = (await nativeServerInventory(request)).find(row => row.node_id === 1 && row.service_type === kind); + expect(original).toBeDefined(); + expect(original.pid_node_id).toBe(original.node_id); + if (kind === 'access-server') expect(original.health).toBe('up'); + expect(original.pid).toBeGreaterThan(1); + const actual = (await nativeServices(0))[original.id]; + expect(actual?.pid).toBe(original.pid); + expect(actual.operation_id).toEqual(expect.any(String)); + expect(await observedNativeProcessAlive(0, original.pid)).toBe(true); + const siblings = await siblingIdentities(original.id); + const sidebar = page.getByRole('complementary', { name: 'Cluster tree sidebar' }); + await expect(sidebar.getByText(`${prefix}-1`, { exact: true })).toBeVisible(); + if (kind === 'access-server') await expect(sidebar.getByRole('button', { name: 'AS-access-1', exact: true })).toHaveCount(0); + await stopAuxiliary(page, request, original, kind, label, prefix); + await step(`native lifecycle: ${original.id} stopped siblings`, async () => expect(await siblingIdentities(original.id)).toEqual(siblings)); + await startAuxiliary(page, request, original, kind, label, prefix, actual.operation_id); + await step(`native lifecycle: ${original.id} restarted siblings`, async () => expect(await siblingIdentities(original.id)).toEqual(siblings)); +} diff --git a/e2e-test/clients/console-ui/e2e/fixtures/parquetFiles.py b/e2e-test/clients/console-ui/e2e/fixtures/parquetFiles.py index 69d0c619d..6a5024864 100644 --- a/e2e-test/clients/console-ui/e2e/fixtures/parquetFiles.py +++ b/e2e-test/clients/console-ui/e2e/fixtures/parquetFiles.py @@ -15,7 +15,7 @@ def encode(table): print(json.dumps({ "empty": encode(pa.table({"id": pa.array([], type=pa.int64()), "message": pa.array([], type=pa.string())})), - "a": encode(pa.table({"id": [1, 2], "message": ["alpha", "雪"]})), + "a": encode(pa.table({"id": [1, 2], "message": ["alpha", "\u96ea"]})), "b": encode(pa.table({"id": [3, 4], "message": ["beta", "gamma"]})), "bad": encode(pa.table({"id": ["wrong type"], "message": ["reject"]})), })) diff --git a/e2e-test/clients/console-ui/e2e/flows/01-shell-ui-behaviors.spec.ts b/e2e-test/clients/console-ui/e2e/flows/01-shell-ui-behaviors.spec.ts index 263847bbf..8da45850f 100644 --- a/e2e-test/clients/console-ui/e2e/flows/01-shell-ui-behaviors.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/01-shell-ui-behaviors.spec.ts @@ -4,10 +4,10 @@ import { test, expect } from '../fixtures/realBackend'; import { diskMenu } from '../fixtures/capacity'; import { step } from '../fixtures/stepTimer'; -// Baseline: 6.0s (2026-10-15). +// Baseline: 6.5s (2026-10-31). test('shell defaults and tree interactions distinguish live and stopped release nodes', async ({ page, request }) => { test.setTimeout(90_000); - const stopped = await request.post('/api/nodes/4/server/stop'); + const stopped = await step('shell KV stop response', () => request.post('/api/nodes/4/server/stop')); expect(stopped.ok(), await stopped.text()).toBe(true); await expect.poll(async () => { const response = await request.get('/api/node/status'); @@ -21,14 +21,24 @@ test('shell defaults and tree interactions distinguish live and stopped release return nodes.filter((node: { id: number; kv_server?: { health: string } }) => [1, 2, 3].includes(node.id) && node.kv_server?.health === 'up').length; }, { timeout: 10_000, intervals: [100] }).toBe(3); - const store = await request.post('/api/stores', { data: { store_id: 207, nodes: [1, 2] } }); + const store = await step('shell Store mutation response', () => request.post('/api/stores', { data: { store_id: 207, nodes: [1, 2] } })); expect(store.status(), await store.text()).toBe(201); const diskInventoryRequests: number[] = []; + const diskInventoryCycles = new Map(); + let topologyRefresh = 0; page.on('request', request => { - const match = new URL(request.url()).pathname.match(/^\/api\/nodes\/(\d+)\/disk-groups$/); - if (match) diskInventoryRequests.push(Number(match[1])); + const path = new URL(request.url()).pathname; + if (path === '/api/racks') topologyRefresh++; + const match = path.match(/^\/api\/nodes\/(\d+)\/disk-groups$/); + if (match) { + const nodeId = Number(match[1]); + diskInventoryRequests.push(nodeId); + const cycle = diskInventoryCycles.get(topologyRefresh) ?? []; + cycle.push(nodeId); + diskInventoryCycles.set(topologyRefresh, cycle); + } }); - await page.goto('/?domain=KV'); + await step('shell DOM setup', () => page.goto('/?domain=KV')); const sidebar = page.getByRole('complementary', { name: 'Cluster tree sidebar' }); await sidebar.getByRole('button', { name: 'datacenter', exact: true }).click({ button: 'right' }); await page.getByRole('menuitem', { name: 'Add Store', exact: true }).click(); @@ -108,7 +118,10 @@ test('shell defaults and tree interactions distinguish live and stopped release const node = sidebar.getByTestId('tree-node-N-3').locator(':scope > [role="treeitem"]'); await node.getByRole('button', { name: 'N-3', exact: true }).click(); await expect(node).toHaveAttribute('aria-selected', 'true'); - await expect.poll(() => diskInventoryRequests, { intervals: [100] }).toEqual([3]); + await expect.poll(() => diskInventoryRequests.length, { intervals: [100] }).toBeGreaterThan(0); + // A topology poll can refresh the selected branch before the DOM assertion + // returns. Each observation cycle still requests exactly this one node once. + for (const cycle of diskInventoryCycles.values()) expect(cycle).toEqual([3]); const storage = sidebar.getByTestId('tree-node-STORAGE-3').locator(':scope > [role="treeitem"]'); await expect(storage).toHaveAttribute('aria-expanded', 'false'); await expect(page.getByRole('region', { name: 'Topology', exact: true })).toBeVisible(); diff --git a/e2e-test/clients/console-ui/e2e/flows/11-cluster-diskdb-lifecycle.spec.ts b/e2e-test/clients/console-ui/e2e/flows/11-cluster-diskdb-lifecycle.spec.ts index 51357c536..af03a5d1b 100644 --- a/e2e-test/clients/console-ui/e2e/flows/11-cluster-diskdb-lifecycle.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/11-cluster-diskdb-lifecycle.spec.ts @@ -5,6 +5,7 @@ import { test, expect } from '../fixtures/realBackend'; import { diskMenu } from '../fixtures/capacity'; import { nativeProcessAlive } from '../fixtures/nativeProcess'; import { liveService, service, serviceRow, stoppedService, unchangedService, type LiveService } from '../fixtures/serviceLifecycle'; +import { step } from '../fixtures/stepTimer'; // Baseline: 15.9s (2026-10-12). test('full deploy flow: deploy diskdb via UI, restart, stop, delete via context menu', async ({ page, request }) => { @@ -100,11 +101,22 @@ async function diskDbLifecycle(page: Page, request: APIRequestContext, kv: LiveS } async function kvLifecycle(page: Page, request: APIRequestContext, original: LiveService, ddb: LiveService) { + const before = await request.get('/api/node/status'); + expect(before.ok(), await before.text()).toBe(true); + const authority = await before.json(); + expect(authority).toMatchObject({ phase: 'active', available: true, cluster_id: expect.any(String), operation_id: expect.any(String) }); await act(page, original, 'Stop CrowDB Storage', '/api/nodes/1/server/stop'); await stoppedService(request, original); await unchangedService(request, ddb); await expect(serviceRow(page, original).getByTitle('Healthy', { exact: true })).toHaveCount(0, { timeout: 10_000 }); await expect(serviceRow(page, ddb).getByTitle('Healthy', { exact: true })).toBeVisible({ timeout: 10_000 }); + // The stopped process may have been Group 0's leader. Confirm its successor + // before submitting the one restart mutation, using the election budget. + await step('KV lifecycle: surviving Group 0 authority', () => expect.poll(async () => { + const response = await request.get('/api/node/status'); + expect(response.ok(), await response.text()).toBe(true); + return response.json(); + }, { timeout: 10_000, intervals: [100] }).toEqual(authority)); await act(page, original, 'Restart CrowDB Storage', '/api/nodes/1/server/restart'); const resumed = await liveService(request, 'paxos-kv', original.pid); await unchangedService(request, ddb); diff --git a/e2e-test/clients/console-ui/e2e/flows/11-cluster-server-lifecycle.spec.ts b/e2e-test/clients/console-ui/e2e/flows/11-cluster-server-lifecycle.spec.ts index 79b3bfefe..8c3544ebc 100644 --- a/e2e-test/clients/console-ui/e2e/flows/11-cluster-server-lifecycle.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/11-cluster-server-lifecycle.spec.ts @@ -6,79 +6,27 @@ import { observeDeployingSteps } from '../fixtures/servicePlanObservation'; import { step } from '../fixtures/stepTimer'; import { observeDefaultDeployments } from '../fixtures/defaultDeployments'; import { nativeProcessAlive } from '../fixtures/nativeProcess'; +import { accessDependencyReady, cycleAuxiliaryService, nativeServerInventory } from '../fixtures/nativeServiceLifecycle'; -// Baseline: 20.7s (2026-10-29), including external stop/start observation without Refresh. +// Baseline: 9.7s (2026-11-01), original combined 3s assertions on the independent-observation image. +// Previous: 20.7s (2026-10-29), including external stop/start observation without Refresh. test('native diagnostics: auxiliary menus stop and restart the actual typed process', async ({ page, request }) => { const alive = (pid: number) => nativeProcessAlive(0, pid); - const list = async () => { - const response = await request.get('/api/servers'); - expect(response.ok(), await response.text()).toBe(true); - return response.json(); - }; + const list = () => nativeServerInventory(request); await page.goto('/?domain=Cluster'); - const sidebar = page.getByRole('complementary', { name: 'Cluster tree sidebar' }); await expect(page.locator('header').getByTitle('Cluster health: Healthy', { exact: true })).toBeVisible(); for (const [kind, label, prefix] of [['chunkdb', 'crowdb-chunk-db', 'CDB'], ['diskio', 'crowdb-disk-io', 'DIO'], ['chunk-kv', 'crowdb-chunk-kv', 'CKV'], ['access-server', 'crowdb-access-server', 'AS']]) { - const original = (await list()).find((row: { node_id: number; service_type: string }) => row.node_id === 1 && row.service_type === kind); - expect(original.pid_node_id).toBe(original.node_id); - if (kind === 'access-server') expect(original.health).toBe('up'); - expect(original.pid).toBeGreaterThan(0); expect(await alive(original.pid)).toBe(true); - const node = sidebar.getByRole('button', { name: 'N-1', exact: true }); - const item = sidebar.getByText(`${prefix}-1`, { exact: true }); - await expect(item).toBeVisible(); - if (kind === 'access-server') await expect(sidebar.getByRole('button', { name: 'AS-access-1', exact: true })).toHaveCount(0); - await node.click({ button: 'right' }); - await page.getByRole('menuitem', { name: `${prefix}-1 Running`, exact: true }).hover(); - const stopped = page.waitForResponse(response => - new URL(response.url()).pathname === `/api/services/${original.id}/stop` - && response.request().method() === 'POST'); - await page.getByRole('menuitem', { name: `Stop ${label}`, exact: true }).click(); - const stopResponse = await stopped; - expect(stopResponse.status(), await stopResponse.text()).toBe(200); - await expect(page.locator('header').getByRole('button', { name: 'Refresh', exact: true }).locator('svg')).not.toHaveClass(/tw-animate-spin/); - await expect.poll(() => alive(original.pid), { intervals: [100] }).toBe(false); - await expect.poll(async () => { - const stopped = (await list()).find((row: { id: string }) => row.id === original.id); - return { id: stopped?.id, pid: stopped?.pid ?? null, ...(kind === 'access-server' ? { health: stopped?.health } : {}) }; - }, { intervals: [100] }).toEqual({ id: original.id, pid: null, ...(kind === 'access-server' ? { health: 'down' } : {}) }); - let startSubmitted = false; - try { - await node.click({ button: 'right' }); - await page.getByRole('menuitem', { name: `${prefix}-1 Stopped`, exact: true }).hover(); - // Native recovery previously reached its listener after 3.15s; keep a - // modest response budget while process and DOM assertions remain 3s. - const started = page.waitForResponse(response => - new URL(response.url()).pathname === `/api/services/${original.id}/restart` - && response.request().method() === 'POST', { timeout: 5_000 }); - const startAt = performance.now(); - startSubmitted = true; - const response = await step(`native lifecycle: ${original.id} Start response`, async () => { - await page.getByRole('menuitem', { name: `Start ${label}`, exact: true }).click(); - return started; - }); - console.log(`[START] ${original.id}: ${(performance.now() - startAt).toFixed(0)}ms`); - expect(response.status(), await response.text()).toBe(200); - await step(`native lifecycle: ${original.id} new process`, () => expect.poll(async () => { - const current = (await list()).find((row: { id: string }) => row.id === original.id); - return current?.pid && current.pid !== original.pid && await alive(current.pid) && (kind !== 'access-server' || current.health === 'up'); - }, { intervals: [100] }).toBe(true)); - expect((await list()).filter((row: { id: string }) => row.id === original.id)).toHaveLength(1); - } finally { - const current = (await list()).find((row: { id: string }) => row.id === original.id); - if (!startSubmitted && !current?.pid) expect((await request.post(`/api/services/${original.id}/restart`, { data: {} })).ok()).toBe(true); - } + await cycleAuxiliaryService(page, request, kind, label, prefix); } await step('native lifecycle: observe external service changes without browser refresh', async () => { const access = (await list()).find((row: { node_id: number; service_type: string }) => row.node_id === 1 && row.service_type === 'access-server'); const stopped = await request.post(`/api/services/${access.id}/stop`, { data: {} }); expect(stopped.ok(), await stopped.text()).toBe(true); - try { - await expect.poll(() => alive(access.pid), { intervals: [100] }).toBe(false); - await expect(page.locator('header').getByTitle('Cluster health: Failed', { exact: true })).toBeVisible(); - } finally { - const started = await request.post(`/api/services/${access.id}/restart`, { data: {} }); - expect(started.ok(), await started.text()).toBe(true); - } + await expect.poll(() => alive(access.pid), { intervals: [100] }).toBe(false); + await expect(page.locator('header').getByTitle('Cluster health: Failed', { exact: true })).toBeVisible(); + await accessDependencyReady(request, access.node_id); + const started = await request.post(`/api/services/${access.id}/restart`, { data: {} }); + expect(started.ok(), await started.text()).toBe(true); await expect(page.locator('header').getByTitle('Cluster health: Healthy', { exact: true })).toBeVisible(); }); expect(await list()).toHaveLength(18); @@ -147,9 +95,13 @@ test('native diagnostics: interrupted six-service plan reconciles without duplic steps: Object.fromEntries(kinds.map(kind => [kind, { state: kind === 'paxos-kv' ? 'deployed' : kind === 'chunkdb' ? 'deploying' : 'pending' }])), } }); expect(saved.ok(), await saved.text()).toBe(true); - let deploymentRequests = 0; + const lifecycleRequests: string[] = []; page.on('request', current => { - if (current.method() === 'POST' && /\/api\/nodes\/\d+\/(server|diskdb|services)\/deploy$/.test(new URL(current.url()).pathname)) deploymentRequests++; + const path = new URL(current.url()).pathname; + if (current.method() === 'POST' && ( + /\/api\/nodes\/\d+\/(server|diskdb|services)\/(deploy|start|restart)$/.test(path) + || /\/api\/services\/[^/]+\/(start|restart)$/.test(path) + )) lifecycleRequests.push(path); }); const dialog = page.getByRole('dialog', { name: 'Node 1 services', exact: true }); const open = async () => { @@ -163,10 +115,9 @@ test('native diagnostics: interrupted six-service plan reconciles without duplic await page.goto('/?domain=Cluster'); await page.reload(); await open(); - await expect(dialog).toContainText('Deployment was interrupted'); await expect(dialog.getByRole('listitem')).toHaveCount(6); - await dialog.getByRole('button', { name: 'Retry failed services', exact: true }).click(); - await expect(dialog.getByRole('button', { name: 'Done', exact: true })).toBeEnabled(); + await expect(dialog.getByRole('button', { name: 'Done', exact: true })).toBeEnabled({ timeout: 3000 }); + await expect(dialog).not.toContainText('Deployment was interrupted'); }); await step('native plan authoritative reconciliation', async () => { await expect.poll(async () => { @@ -177,14 +128,14 @@ test('native diagnostics: interrupted six-service plan reconciles without duplic const current = await request.get('/api/servers'); expect(current.ok()).toBe(true); expect(identity(await current.json())).toEqual(identity(original)); - expect(deploymentRequests).toBe(0); + expect(lifecycleRequests).toEqual([]); }); await step('native completed plan reload', async () => { await page.reload(); await open(); await expect(dialog.getByRole('button', { name: 'Done', exact: true })).toBeEnabled(); await expect(dialog).not.toContainText('Deployment was interrupted'); - expect(deploymentRequests).toBe(0); + expect(lifecycleRequests).toEqual([]); }); }); diff --git a/e2e-test/clients/console-ui/e2e/flows/12-bridge-discovery.spec.ts b/e2e-test/clients/console-ui/e2e/flows/12-bridge-discovery.spec.ts index a7b3d91d4..8088015b6 100644 --- a/e2e-test/clients/console-ui/e2e/flows/12-bridge-discovery.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/12-bridge-discovery.spec.ts @@ -26,7 +26,7 @@ test('Docker default setup: submit storage and services then close the initiatin for (const checkbox of await dialog.getByRole('region', { name: 'Node service choices' }).getByRole('checkbox').all()) await expect(checkbox).toBeChecked(); await expect(dialog.getByLabel('Managed file capacity (GiB)', { exact: true })).toHaveValue('16'); await expect(dialog.getByLabel('Managed file capacity (GiB)', { exact: true })).toBeDisabled(); - await expect(dialog.getByLabel('crowdb-disk-db RPC port', { exact: true })).toHaveValue('7100'); + await expect(dialog.getByLabel('crowdb-disk-db RPC port', { exact: true })).toHaveValue('11200'); await expect(dialog.getByLabel('crowdb-disk-db RPC port', { exact: true })).toBeDisabled(); await dialog.getByRole('button', { name: 'Join and start', exact: true }).click(); await expect(dialog).toHaveCount(0, { timeout: 10_000 }); @@ -58,8 +58,8 @@ test('Docker default setup: progress is visible in the node properties', async ( const groups = (await usage.json()).disk_groups; expect(groups.map((group: { disk_group_id: number }) => group.disk_group_id)).toContain(1); const disk = groups.find((group: { disk_group_id: number }) => group.disk_group_id === 1).disks[0]; - expect(disk.zone_count).toBe(64); - expect(disk.zone_size_units * disk.unit_size_bytes).toBe(256 * 1024 * 1024); + expect(disk.zone_count).toBe(1); + expect(disk.zone_size_units * disk.unit_size_bytes).toBe(16 * 1024 ** 3); const bitmap = await page.request.get(`${process.env.CROWDB_NODE_UI_ORIGIN}/api/diskdb/usage?dg=1&disk=${disk.disk_id}&zone=0`); expect(bitmap.ok(), await bitmap.text()).toBe(true); expect((await bitmap.json()).disk_groups[0].disks[0].zone_usages[0].usage_bitmap).toMatch(/^[0-9a-f]+$/); diff --git a/e2e-test/clients/console-ui/e2e/flows/13-node-service-plans.spec.ts b/e2e-test/clients/console-ui/e2e/flows/13-node-service-plans.spec.ts index 86b9fc18b..9c88a35b9 100644 --- a/e2e-test/clients/console-ui/e2e/flows/13-node-service-plans.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/13-node-service-plans.spec.ts @@ -3,7 +3,7 @@ import type { APIRequestContext } from '@playwright/test'; import { test, expect } from '../fixtures/realBackend'; import { diskMenu } from '../fixtures/capacity'; -import { nativeProcessAlive } from '../fixtures/nativeProcess'; +import { nativeProcessAlive, nativeServices } from '../fixtures/nativeProcess'; const kinds = ['paxos-kv', 'diskdb', 'chunkdb', 'diskio', 'chunk-kv', 'access-server']; async function json(request: APIRequestContext, path: string) { @@ -70,6 +70,19 @@ test('native plans: fixed ports isolate three nodes and failed DiskDB retries pr const retained = await json(request, '/api/service-plans'); expect(retained[1].overrides).toEqual(before[1].overrides); expect(retained[1].steps.diskdb).toEqual(before[1].steps.diskdb); + // Keep the failed registration for explicit Retry, but cancel its running + // intent before freeing the port so monitor recovery cannot win this phase. + const stopped = await request.post('/api/services/diskdb-1/stop'); + expect(stopped.ok(), await stopped.text()).toBe(true); + expect(await stopped.json()).toMatchObject({ id: 'diskdb-1', pid: null, removed: false }); + // Monitor status lists owned children; Stop removes the child but retains registration. + await expect.poll(async () => Object.hasOwn(await nativeServices(0), 'diskdb-1'), { intervals: [100] }).toBe(false); + const registered = (await json(request, '/api/servers')).filter((server: { id: string }) => server.id === 'diskdb-1'); + expect(registered).toHaveLength(1); + expect(registered[0]).toMatchObject({ id: 'diskdb-1', node_id: 1, service_type: 'diskdb' }); + expect(registered[0].pid).toBeFalsy(); + expect((await json(request, '/api/service-plans'))[1].steps.diskdb.state).toBe('failed'); + expect(await nativeProcessAlive(0, guard.pid)).toBe(true); const removed = await request.delete(`/api/services/${guard.id}`); expect(removed.ok(), await removed.text()).toBe(true); await expect.poll(() => nativeProcessAlive(0, guard.pid), { intervals: [100] }).toBe(false); diff --git a/e2e-test/clients/console-ui/e2e/flows/30-kv-ops-basic.spec.ts b/e2e-test/clients/console-ui/e2e/flows/30-kv-ops-basic.spec.ts index 599278707..4d53c2120 100644 --- a/e2e-test/clients/console-ui/e2e/flows/30-kv-ops-basic.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/30-kv-ops-basic.spec.ts @@ -101,7 +101,7 @@ test.describe('kv ops · put/get/scan/delete', () => { await openKvPanel(page); const table = page.getByTestId('kv-scan-table'); await expect(table).toContainText('field00FFend'); - await expect(table).toContainText('helloFE8001中文'); + await expect(table).toContainText('helloFE8001\u4e2d\u6587'); await expect(table).not.toContainText('0x'); await expect(table.locator('[data-byte-format="hex"]')).toHaveText(['00FF', 'FE8001']); for (const run of await table.locator('[data-byte-format="hex"]').all()) { @@ -113,7 +113,7 @@ test.describe('kv ops · put/get/scan/delete', () => { } await table.getByText('field', { exact: true }).click(); const detail = page.getByRole('region', { name: 'Selected key' }); - await expect(detail).toContainText('helloFE8001中文'); + await expect(detail).toContainText('helloFE8001\u4e2d\u6587'); await expect(detail.locator('[data-byte-format="hex"]')).toHaveText(['00FF', 'FE8001']); for (const run of await detail.locator('[data-byte-format="hex"]').all()) { await expect(run).toHaveCSS('color', 'rgb(133, 133, 133)'); diff --git a/e2e-test/clients/console-ui/e2e/flows/55-chunk-kv-catalog.spec.ts b/e2e-test/clients/console-ui/e2e/flows/55-chunk-kv-catalog.spec.ts index e8d89e8c2..1769b114b 100644 --- a/e2e-test/clients/console-ui/e2e/flows/55-chunk-kv-catalog.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/55-chunk-kv-catalog.spec.ts @@ -1,28 +1,33 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -// Baseline: partition/overlay 1.4s, unavailable 0.208s, journal 0.534s, placement 0.321s (2026-10-03). +// Baseline: journal 5.3s (2026-11-03); partition/overlay 1.4s, unavailable 0.208s, placement 0.321s (2026-10-03). import { test, expect } from '../fixtures/realBackend'; +import { step } from '../fixtures/stepTimer'; +import { journalRuntimeResponse, type JournalRuntimeTarget } from '../fixtures/journalRuntimeRequest'; import type { APIRequestContext, APIResponse, Page, Response } from '@playwright/test'; -import type { CatalogPage, Partition } from '../../src/chunk-kv/catalog'; +import { identity, type CatalogPage, type Partition } from '../../src/chunk-kv/catalog'; test('native diagnostics: journal identity survives Chunk navigation and owner interruption', async ({ page, request }) => { - const bucket = `journal-${process.pid}-${Date.now()}`; + const bucket = process.env.CROWDB_NATIVE_JOURNAL_BUCKET; + if (!bucket) throw new Error("Journal diagnostics require the settled owned object"); const object = `/api/access/s3/${bucket}/value`; try { - expect((await request.put(`/api/access/s3/${bucket}`)).ok()).toBe(true); - expect((await request.put(object, { data: 'native journal observation' })).ok()).toBe(true); + const witness = await step('journal fixture witness', () => request.get(object)); + expect(witness.ok(), await witness.text()).toBe(true); + expect(await witness.text()).toBe('native journal observation'); const catalogObservation = page.waitForResponse(response => new URL(response.url()).pathname === '/api/chunk-kv/catalog'); - await page.goto('/?domain=Chunk-KV'); + await step('journal DOM setup', () => page.goto('/?domain=Chunk-KV')); const response = await catalogObservation; expect(response.ok(), await response.text()).toBe(true); - const { partition, observed } = await selectActiveJournal(page, request, await response.json()); + const { partition, observed } = await step('journal initial lifecycle', async () => selectActiveJournal(page, request, await response.json())); const runtime = await observed.json(); expect(runtime.journal.chunk_record).not.toBeNull(); await page.getByRole('tab', { name: 'Journal', exact: true }).click(); const journal = page.getByRole('tabpanel', { name: 'Journal', exact: true }); - await expect(journal).toContainText(runtime.journal.chunk_record.chunk_id); + const storage = journal.getByRole('region', { name: 'Journal storage', exact: true }); + await expect(storage).toContainText(runtime.journal.chunk_record.chunk_id); for (const [name, value] of [['Stream chunk record generation', runtime.journal.generation], ['Stream writer epoch', runtime.journal.writer_epoch], ['Metadata group', runtime.journal.metadata_group_id]]) { - await expect(journal.locator('dt').filter({ hasText: new RegExp(`^${name}$`) }).locator('..').locator('dd')).toHaveText(value); + await expect(storage.locator('dt').filter({ hasText: new RegExp(`^${name}$`) }).locator('..').locator('dd')).toHaveText(value); } await expect(journal.getByLabel('Stream chunk record', { exact: true })).toContainText(runtime.journal.chunk_record.logical_end); await journal.getByRole('button', { name: 'Inspect Chunk', exact: true }).click(); @@ -36,26 +41,28 @@ test('native diagnostics: journal identity survives Chunk navigation and owner i await page.getByRole('button', { name: 'Refresh catalog', exact: true }).click(); const currentResponse = await currentMap; expect(currentResponse.ok(), await currentResponse.text()).toBe(true); - const current = await selectActiveJournal(page, request, await currentResponse.json(), partition.id); + const current = await step('journal current owner lifecycle', async () => selectActiveJournal(page, request, await currentResponse.json(), partition.id)); expect(current.partition.id).toBe(partition.id); await page.getByRole('tab', { name: 'Journal', exact: true }).click(); await expect(page.getByRole('region', { name: 'Partition runtime', exact: true }).getByRole('button', { name: 'Refresh runtime', exact: true })).toBeEnabled(); const owner = `chunk-kv-${current.partition.owner_id}`; - expect((await request.post(`/api/services/${owner}/stop`, { data: {} })).ok()).toBe(true); + expect((await step('journal owner stop response', () => request.post(`/api/services/${owner}/stop`, { data: {} }))).ok()).toBe(true); try { const observation = page.getByRole('region', { name: 'Partition runtime', exact: true }); - await observation.getByRole('button', { name: 'Refresh runtime', exact: true }).click(); - await expect(observation).toContainText('Runtime unavailable'); - await expect(journal.getByLabel('Stream chunk record', { exact: true })).toHaveCount(0); - await expect(journal.getByRole('button', { name: 'Inspect Chunk', exact: true })).toHaveCount(0); + await step('journal owner loss DOM refresh', async () => { + await observation.getByRole('button', { name: 'Refresh runtime', exact: true }).click(); + await expect(observation).toContainText('Runtime unavailable'); + await expect(journal.getByLabel('Stream chunk record', { exact: true })).toHaveCount(0); + await expect(journal.getByRole('button', { name: 'Inspect Chunk', exact: true })).toHaveCount(0); + }); } finally { - expect((await request.post(`/api/services/${owner}/restart`, { data: {} })).ok()).toBe(true); + expect((await step('journal owner restart response', () => request.post(`/api/services/${owner}/restart`, { data: {} }))).ok()).toBe(true); } const refreshedMap = page.waitForResponse(response => new URL(response.url()).pathname === '/api/chunk-kv/catalog'); await page.getByRole('button', { name: 'Refresh catalog', exact: true }).click(); const mapResponse = await refreshedMap; expect(mapResponse.ok(), await mapResponse.text()).toBe(true); - const restored = await selectActiveJournal(page, request, await mapResponse.json(), partition.id); + const restored = await step('journal restored owner lifecycle', async () => selectActiveJournal(page, request, await mapResponse.json(), partition.id)); expect(restored.partition.id).toBe(partition.id); await page.getByRole('tab', { name: 'Journal', exact: true }).click(); const observation = page.getByRole('region', { name: 'Partition runtime', exact: true }); @@ -67,9 +74,9 @@ test('native diagnostics: journal identity survives Chunk navigation and owner i expect((await request.get(stale.toString())).status()).toBe(409); } finally { try { - expect((await request.delete(object)).status()).toBe(204); + expect((await step('journal object teardown', () => request.delete(object))).status()).toBe(204); } finally { - expect((await request.delete(`/api/access/s3/${bucket}`)).status()).toBe(204); + expect((await step('journal bucket teardown', () => request.delete(`/api/access/s3/${bucket}`))).status()).toBe(204); } } }); @@ -175,12 +182,39 @@ test('native diagnostics: Chunk-KV graph survives tab changes, refresh and resiz }); }); +async function refreshSelectedJournal(page: Page, target: JournalRuntimeTarget, remaining: () => number) { + const properties = page.getByRole('region', { name: 'Split properties', exact: true }); + const property = (name: string) => properties.locator('dt').filter({ hasText: new RegExp(`^${name}$`) }).locator('..').locator('dd').textContent({ timeout: remaining() }); + const sameSelection = await properties.count() === 1 && (await property('Split ID')) === target.id + && (await property('Owner epoch')) === target.epoch && (await property('Generation')) === target.generation; + const select = () => page.getByLabel('Partition range map').getByRole('button', { name: `Partition ${target.id}`, exact: true }).click({ timeout: remaining() }); + const refreshRuntime = page.getByRole('region', { name: 'Partition runtime', exact: true }).getByRole('button', { name: 'Refresh runtime', exact: true }); + if (!sameSelection) { + const observed = await step('journal automatic selection response', () => journalRuntimeResponse(page, target, select, remaining)); + // A changed identity starts its own observation. Preserve that response, + // including a catalog conflict, for the caller's coherence checks. + return { observed, refreshRuntime }; + } + await expect(refreshRuntime).toBeEnabled({ timeout: remaining() }); + // The same partition identity does not trigger a new observation. Explicitly + // refresh runtime so owner restart is observed even without a new catalog. + const observed = await step('journal runtime DOM refresh', () => journalRuntimeResponse(page, target, + () => refreshRuntime.click({ timeout: remaining() }), remaining)); + return { observed, refreshRuntime }; +} + async function selectActiveJournal(page: Page, request: APIRequestContext, initial: CatalogPage, partitionId?: string) { let catalog = initial; + const deadline = performance.now() + 3_000; + const remaining = () => { + const budget = deadline - performance.now(); + if (budget <= 0) throw new Error('Journal observation exceeded its three-second deadline'); + return budget; + }; let selected: { partition: Partition; observed: Response } | null = null; const refresh = async (action: () => Promise) => { const observation = page.waitForResponse(response => new URL(response.url()).pathname === '/api/chunk-kv/catalog'); - await action(); + await step('journal catalog refresh response', action); const response = await observation; expect(response.ok(), await response.text()).toBe(true); catalog = await response.json(); @@ -206,18 +240,23 @@ async function selectActiveJournal(page: Page, request: APIRequestContext, initi } if (entry.state !== 'Serving') continue; const query = new URLSearchParams({ id: entry.id, epoch: entry.epoch, generation: catalog.generation, page: String(catalog.page), offset: String(catalog.offset) }); - const response = await request.get(`/api/chunk-kv/runtime?${query}`); + const response = await step('journal runtime API response', () => request.get(`/api/chunk-kv/runtime?${query}`)); if (await changed(response)) return false; expect(response.ok(), await response.text()).toBe(true); if (!(await response.json()).journal.chunk_record) continue; - await page.getByLabel('Partition range map').getByRole('button', { name: `Partition ${entry.id}`, exact: true }).click(); - // Selecting the same partition leaves its identity unchanged. Explicitly - // refresh runtime so owner restart is observed even without a new catalog. - const observation = page.waitForResponse(response => new URL(response.url()).pathname === '/api/chunk-kv/runtime'); - await page.getByRole('region', { name: 'Partition runtime', exact: true }).getByRole('button', { name: 'Refresh runtime', exact: true }).click(); - const observed = await observation; + const target = { origin: new URL(page.url()).origin, id: entry.id, epoch: entry.epoch, + generation: catalog.generation, page: catalog.page, offset: catalog.offset }; + const { observed, refreshRuntime } = await refreshSelectedJournal(page, target, remaining); if (await changed(observed)) return false; expect(observed.ok(), await observed.text()).toBe(true); + const runtime = await observed.json(); + expect(runtime.partition_id).toBe(entry.id); + expect(runtime.owner_epoch).toBe(entry.epoch); + expect(runtime.catalog_generation).toBe(catalog.generation); + expect(runtime.stream_id).toBe(identity(entry.artifact.stream_name)); + expect(runtime.journal?.offset).toBe(0); + expect(runtime.journal?.chunk_record).toBeTruthy(); + await expect(refreshRuntime).toBeEnabled({ timeout: remaining() }); selected = { partition: entry, observed }; return true; } diff --git a/e2e-test/clients/console-ui/e2e/flows/71-s3-native.spec.ts b/e2e-test/clients/console-ui/e2e/flows/71-s3-native.spec.ts index 96e53ddfb..3b49d4635 100644 --- a/e2e-test/clients/console-ui/e2e/flows/71-s3-native.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/71-s3-native.spec.ts @@ -9,7 +9,7 @@ import { bucketFailure, bucketReply } from '../fixtures/bucketDiagnostics'; // Baseline: 5.3s (2026-10-04); normal three-node fixture, actual multipart acceptance. test('S3 native multipart upload, HEAD, bounded preview and full round trip', async ({ page, request }) => { const bucket = `console-multipart-${process.pid}-${Date.now()}`; - const key = 'multipart 中文.txt'; + const key = 'multipart \u4e2d\u6587.txt'; const bucketPath = `/api/access/s3/${bucket}`; const objectPath = `${bucketPath}/${encodeURIComponent(key)}`; const bytes = Buffer.alloc(9 * 1024 * 1024, 'x'); diff --git a/e2e-test/clients/console-ui/e2e/flows/92-three-node-data.spec.ts b/e2e-test/clients/console-ui/e2e/flows/92-three-node-data.spec.ts index 0ef1a7041..d1b0f0a4e 100644 --- a/e2e-test/clients/console-ui/e2e/flows/92-three-node-data.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/92-three-node-data.spec.ts @@ -12,6 +12,8 @@ import { disklessExpansion, verifyDisklessSlotBalance } from '../fixtures/diskle import { execFile } from 'node:child_process'; import { promisify } from 'node:util'; import { resolve } from 'node:path'; +import { parseIcebergJson } from '../../src/iceberg/json'; +import type { TableLoad } from '../../src/iceberg/types'; import type { APIRequestContext, Page } from '@playwright/test'; @@ -165,7 +167,7 @@ async function verifyServices({ page, request, deployments }: UIContext) { async function s3RoundTrip({ page, request }: UIContext) { await step('three-node: S3 upload and exact read through UI', async () => { - const content = 'CROWDB three-node UI round trip\n雪\n'; + const content = 'CROWDB three-node UI round trip\n\u96ea\n'; await page.getByTestId('domain-s3').click(); await resourceAction(page, page.getByRole('navigation', { name: 'S3 buckets' }).getByRole('button', { name: 'S3', exact: true }), 'Create bucket…'); await page.getByLabel('New bucket', { exact: true }).fill('ui-three-node'); @@ -222,7 +224,21 @@ async function icebergRoundTrip({ page, baseURL }: UIContext) { execution.child.stdin!.end(JSON.stringify(writerConfig)); const result = await execution; console.log(result.stdout); if (result.stderr) console.log(result.stderr); - await page.getByRole('button', { name: 'Refresh table', exact: true }).click(); + await step('three-node: refreshed Iceberg metadata reaches the DOM', async () => { + const refreshed = page.waitForResponse(response => response.request().method() === 'GET' + && new URL(response.url()).pathname === '/api/access/iceberg/v1/namespaces/ui_three_node/tables/events'); + const refresh = page.getByRole('button', { name: 'Refresh table', exact: true }); + await refresh.click(); + const response = await refreshed; + expect(response.status(), await response.text()).toBe(200); + const loaded = parseIcebergJson(await response.text()) as TableLoad; + const current = String(loaded.metadata['current-snapshot-id']); + const snapshot = loaded.metadata.snapshots?.find(snapshot => String(snapshot['snapshot-id']) === current); + expect(snapshot?.summary).toMatchObject({ 'total-records': '3' }); + await expect(page.getByRole('table', { name: 'Table snapshots', exact: true }).getByRole('row') + .filter({ has: page.getByRole('button', { name: current, exact: true }) })).toContainText('Current'); + await expect(refresh).toBeEnabled(); + }); await expect(page.getByRole('button', { name: 'Inspect current snapshot', exact: true })).toBeVisible(); await page.getByRole('button', { name: 'Inspect current snapshot', exact: true }).click(); await expect(page.getByRole('table', { name: 'Manifest list records' })).toContainText('avro'); diff --git a/e2e-test/clients/iceberg-python/listing.py b/e2e-test/clients/iceberg-python/listing.py index 0ca7624c0..c16a01a58 100644 --- a/e2e-test/clients/iceberg-python/listing.py +++ b/e2e-test/clients/iceberg-python/listing.py @@ -28,7 +28,7 @@ def main(): endpoint_override=endpoint.netloc, force_virtual_addressing=False, ) prefix = f'{config["bucket"]}/{config["prefix"]}data/' - for suffix in ["a.json", "nested/b.json", "雪&<>+%.json"]: + for suffix in ["a.json", "nested/b.json", "\u96ea&<>+%.json"]: path = prefix + suffix file = PyArrowFile("s3://" + path, path, filesystem) assert not file.exists() @@ -39,7 +39,7 @@ def main(): assert stream.read() == b"{}" selected = filesystem.get_file_info(fs.FileSelector(prefix.removesuffix("/"), recursive=True)) names = sorted(file.path for file in selected if file.type == fs.FileType.File) - assert names == sorted(prefix + suffix for suffix in ["a.json", "nested/b.json", "雪&<>+%.json"]), names + assert names == sorted(prefix + suffix for suffix in ["a.json", "nested/b.json", "\u96ea&<>+%.json"]), names print(f"PyIceberg {pyiceberg.__version__} / PyArrow {pyarrow.__version__}: exact creation, HEAD/GET and prefix discovery passed") verify(config) if os.environ.get("CROWDB_E2E_NATIVE_LOADER_FLOW") == "1": diff --git a/e2e-test/clients/iceberg-python/namespace.py b/e2e-test/clients/iceberg-python/namespace.py index 0e7ada86a..95fcb6fe0 100644 --- a/e2e-test/clients/iceberg-python/namespace.py +++ b/e2e-test/clients/iceberg-python/namespace.py @@ -75,7 +75,7 @@ def main(): def verify_namespaces(uri, properties): writer = load_catalog("crowdb", **(properties | {"token": "w" * 32})) namespace = ("client-" + uuid.uuid4().hex,) - child = namespace + ("冰+a%2F",) + child = namespace + ("\u51b0+a%2F",) writer.create_namespace(namespace, {"owner": "original"}) assert writer.namespace_exists(namespace) assert writer.load_namespace_properties(namespace) == {"owner": "original"} diff --git a/e2e-test/clients/s3-go/main.go b/e2e-test/clients/s3-go/main.go index 000e0e4eb..eb234210b 100644 --- a/e2e-test/clients/s3-go/main.go +++ b/e2e-test/clients/s3-go/main.go @@ -129,6 +129,24 @@ func capture(action func()) (failure any) { return nil } +func failureCode(failure any) string { + code := fmt.Sprintf("%T", failure) + if assertion, ok := failure.(assertionFailure); ok { + return "assertion: " + string(assertion) + } + if err, ok := failure.(error); ok { + var api smithy.APIError + if errors.As(err, &api) { + code = api.ErrorCode() + } + var operation *smithy.OperationError + if errors.As(err, &operation) { + code = operation.Operation() + ": " + code + } + } + return code +} + func run() int { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() @@ -149,20 +167,10 @@ func run() int { exit = 42 } // SDK errors can contain request URLs; report only the code/type and stage. - code := fmt.Sprintf("%T", failure) - if assertion, ok := failure.(assertionFailure); ok { - code = "assertion: " + string(assertion) - } - if err, ok := failure.(error); ok { - var api smithy.APIError - if errors.As(err, &api) { - code = api.ErrorCode() - } - } - fmt.Fprintf(os.Stderr, "go: FAILED step=%s code=%s\n", s.current, code) + fmt.Fprintf(os.Stderr, "go: FAILED step=%s code=%s\n", s.current, failureCode(failure)) } - if capture(s.cleanup) != nil { - fmt.Fprintln(os.Stderr, "go: cleanup FAILED") + if failure := capture(s.cleanup); failure != nil { + fmt.Fprintf(os.Stderr, "go: cleanup FAILED code=%s\n", failureCode(failure)) exit = 1 } if exit == 0 { diff --git a/e2e-test/clients/s3/batch_delete.py b/e2e-test/clients/s3/batch_delete.py index 6eb1e2d31..bdecf3df7 100644 --- a/e2e-test/clients/s3/batch_delete.py +++ b/e2e-test/clients/s3/batch_delete.py @@ -12,7 +12,7 @@ class BatchDeleteCases: def test_batch_delete_preserves_exact_keys_and_quiet(self): bucket = self.bucket + "-batch" - keys = ["escaped/<&>.bin", "unicode/边界%25+.bin", "literal\rline.bin"] + keys = ["escaped/<&>.bin", "unicode/\u8fb9\u754c%25+.bin", "literal\rline.bin"] self.client.create_bucket(Bucket=bucket) for key in keys + ["retained"]: self.client.put_object(Bucket=bucket, Key=key, Body=b"selected") @@ -35,7 +35,7 @@ def test_batch_delete_thousand_keys_and_unversioned_retry(self): if phase not in ("all", "initial", "retry", "new-publication"): raise ValueError("unknown batch-delete phase") bucket = self.bucket + "-batch-limit" - keys = [f"bulk/{index:04}/<&>边界" for index in range(1000)] + keys = [f"bulk/{index:04}/<&>\u8fb9\u754c" for index in range(1000)] selection = {"Objects": [{"Key": key} for key in keys]} if phase in ("all", "initial"): self.client.create_bucket(Bucket=bucket) diff --git a/e2e-test/clients/s3/default_client.py b/e2e-test/clients/s3/default_client.py index f23de0c47..f12f40937 100644 --- a/e2e-test/clients/s3/default_client.py +++ b/e2e-test/clients/s3/default_client.py @@ -67,7 +67,7 @@ def test_presigned_transfers_tamper_and_expiry(self): client = self.default_client() bucket = self.bucket + "-presign" client.create_bucket(Bucket=bucket) - params = {"Bucket": bucket, "Key": "exact/%25+边界"} + params = {"Bucket": bucket, "Key": "exact/%25+\u8fb9\u754c"} put = client.generate_presigned_url("put_object", Params=params, ExpiresIn=60) self.assertEqual(self.presigned_http("PUT", put, b"presigned bytes")[0], 200) get = client.generate_presigned_url("get_object", Params=params, ExpiresIn=60) diff --git a/e2e-test/clients/s3/object_copy.py b/e2e-test/clients/s3/object_copy.py index a86a708fd..c1046c729 100644 --- a/e2e-test/clients/s3/object_copy.py +++ b/e2e-test/clients/s3/object_copy.py @@ -8,7 +8,7 @@ class CopyCases: def test_server_side_copy_preserves_bytes_and_supported_metadata(self): source_bucket = f"{self.bucket}-copy-source" target_bucket = f"{self.bucket}-copy-target" - source_key = "encoded/雪 %?+&.json" + source_key = "encoded/\u96ea %?+&.json" payload = b'{"copy":"immutable"}' self.client.create_bucket(Bucket=source_bucket) self.client.create_bucket(Bucket=target_bucket) diff --git a/e2e-test/clients/s3/wire.py b/e2e-test/clients/s3/wire.py index d4a87ef74..5fabaad3d 100644 --- a/e2e-test/clients/s3/wire.py +++ b/e2e-test/clients/s3/wire.py @@ -7,7 +7,7 @@ class WireCases: def test_signed_raw_http_wire_contract(self): bucket = f"{self.bucket}-raw" - key = "raw/%25+边界.bin" + key = "raw/%25+\u8fb9\u754c.bin" path = f"/{bucket}/{quote(key, safe='/')}" payload = b"raw-http-object\x00payload" @@ -104,7 +104,7 @@ def test_fragmentation_and_storage_boundaries(self): sizes = [0, 1, 65505, 65506, 65507, 1024 * 1024 + 31, 4 * 1024 * 1024 + 127] keys = [] for index, size in enumerate(sizes): - key = f"encoded/边界-{index}-%00.bin" + key = f"encoded/\u8fb9\u754c-{index}-%00.bin" payload = bytes((offset * 31 + index) % 256 for offset in range(size)) digest = md5(payload) result = client.put_object( diff --git a/e2e-test/console_cli_client.py b/e2e-test/console_cli_client.py index ef6f578df..fa1339482 100644 --- a/e2e-test/console_cli_client.py +++ b/e2e-test/console_cli_client.py @@ -19,7 +19,8 @@ def diagnostics(self): opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) for index, address in enumerate(self.addresses): try: - with opener.open(f"http://{address}:9091/_crowdb/metrics", timeout=2) as response: + port = self.access_listeners(index)["s3_port"] + with opener.open(f"http://{address}:{port}/_crowdb/metrics", timeout=2) as response: metrics = response.read(1024 * 1024 + 1) if len(metrics) > 1024 * 1024: raise ValueError("Access metrics exceeded the diagnostic limit") diff --git a/e2e-test/console_cli_fixture.py b/e2e-test/console_cli_fixture.py index e6b45eb2f..63b4a8922 100644 --- a/e2e-test/console_cli_fixture.py +++ b/e2e-test/console_cli_fixture.py @@ -203,7 +203,7 @@ def refresh(self): "published_console_ports": [self.port_base + index * 10 for index in range(3)]}, indent=2)) def read_owned(self, path, index=0): - if path not in ("node-binding.json", "diskdb-execution"): + if path not in ("node-binding.json", "node-policy.json", "diskdb-execution"): raise ValueError("unsupported CLI recovery observation") if type(index) is not int or not 0 <= index < len(getattr(self, "servers", ())): raise ValueError("CLI recovery node is outside the owned cluster") @@ -216,6 +216,14 @@ def read_owned(self, path, index=0): path = f"services/diskdb-{node_id}/execution.json" return json.loads(self.command("exec", info["Id"], "cat", "/opt/crowdb/data/" + path)) + def access_listeners(self, index): + services = self.read_owned("node-policy.json", index)["setup"]["services"] + selected = [service["listeners"] for service in services + if service["kind"] == "access-server" and service["enabled"]] + if len(selected) != 1: + raise ValueError("owned node needs one accepted Access listener selection") + return selected[0] + def diagnostics(self): if not (self.root / "persistent/oci-clusters" / f"{self.run_id}.json").exists(): return diff --git a/e2e-test/console_cli_io_fault.py b/e2e-test/console_cli_io_fault.py new file mode 100644 index 000000000..09c9e73f2 --- /dev/null +++ b/e2e-test/console_cli_io_fault.py @@ -0,0 +1,91 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Bounded DiskIO packet loss inside verified CLI-owned network namespaces.""" + +from contextlib import ExitStack +import hashlib +import json +from pathlib import Path +import re +import shutil +import subprocess + + +class DiskIoOutage: + def __init__(self, cluster): + self.cluster = cluster + self.chain = "CROWDB_E2E_" + hashlib.sha256(cluster.run_id.encode()).hexdigest()[:12] + self.nodes = [] + for index, name in enumerate(cluster.servers): + policy = cluster.read_owned("node-policy.json", index) + services = [service for service in policy["setup"]["services"] + if service["kind"] == "diskio" and service["enabled"]] + if len(services) != 1: + raise ValueError("fault needs one accepted DiskIO listener per owned node") + port = services[0]["listeners"]["rpc_port"] + if type(port) is not int or not 0 < port < 65536: + raise ValueError("invalid owned DiskIO listener") + self.nodes.append((name, port)) + if len(self.nodes) != 3: + raise ValueError("terminal writer fault requires three owned nodes") + + def _prepare_tools(self): + binary = shutil.which("iptables-nft") + if not binary: + raise RuntimeError("owned network fault requires the host iptables-nft executable") + self.tools = self.cluster.artifacts / "fault-network-tools" + self.tools.mkdir() + shutil.copy2(binary, self.tools / "iptables-nft") + result = subprocess.run(["ldd", binary], capture_output=True, text=True, check=False, timeout=10) + if result.returncode: + raise RuntimeError(f"netfilter dependency inspection failed: {result.stdout}{result.stderr}") + libraries = [Path(path) for path in re.findall(r"(?:=>\s*|^\s*)(/\S+)", result.stdout, re.M)] + loaders = [path for path in libraries if path.name.startswith("ld-linux")] + extensions = [path.parent / "xtables" for path in libraries if path.name.startswith("libxtables")] + if len(loaders) != 1 or len(extensions) != 1 or not extensions[0].is_dir(): + raise RuntimeError(f"cannot locate the host netfilter loader/extensions: {result.stdout}") + self.loader = loaders[0].name + for library in libraries: + shutil.copy2(library, self.tools / library.name) + shutil.copytree(extensions[0], self.tools / "xtables") + files = {str(path.relative_to(self.tools)): hashlib.sha256(path.read_bytes()).hexdigest() + for path in self.tools.rglob("*") if path.is_file()} + (self.cluster.artifacts / "fault-network-tools.json").write_text(json.dumps(files, indent=2)) + + def _command(self, node, *arguments): + name, _port = node + info = self.cluster.inspect_owned(name) + if info["HostConfig"]["NetworkMode"] != "crowdb-" + self.cluster.run_id: + raise ValueError("fault must remain inside the owned CLI bridge namespace") + root = "/run/crowdb-netfilter" + return self.cluster.command("run", "--rm", "--network", "container:" + info["Id"], + "--cap-drop", "ALL", "--cap-add", "NET_ADMIN", "--memory", "64m", "--pids-limit", "32", + "--cpu-shares", "1024", "--label", "org.crowdb.e2e.run=" + self.cluster.run_id, + "--mount", f"type=bind,source={self.tools.resolve()},target={root},readonly", + "--env", f"XTABLES_LIBDIR={root}/xtables", "--entrypoint", f"{root}/{self.loader}", + self.cluster.image, "--library-path", root, f"{root}/iptables-nft", "-w", "2", *arguments) + + def __enter__(self): + self._prepare_tools() + with ExitStack() as cleanup: + for node in self.nodes: + self._command(node, "-N", self.chain) + cleanup.callback(self._command, node, "-X", self.chain) + self._command(node, "-A", self.chain, "-p", "tcp", "--dport", str(node[1]), "-j", "DROP") + cleanup.callback(self._command, node, "-F", self.chain) + self._command(node, "-I", "INPUT", "1", "-j", self.chain) + cleanup.callback(self._command, node, "-D", "INPUT", "-j", self.chain) + self.cleanup = cleanup.pop_all() + return self + + def verify(self): + proof = [] + for node in self.nodes: + self._command(node, "-C", "INPUT", "-j", self.chain) + self._command(node, "-C", self.chain, "-p", "tcp", "--dport", str(node[1]), "-j", "DROP") + proof.append({"container": self.cluster.inspect_owned(node[0])["Id"], + "diskio_port": node[1], "chain": self.chain, "blocked": True}) + (self.cluster.artifacts / "fault-network-blocked.json").write_text(json.dumps(proof, indent=2)) + + def __exit__(self, *exception): + return self.cleanup.__exit__(*exception) diff --git a/e2e-test/console_cli_readiness.py b/e2e-test/console_cli_readiness.py new file mode 100644 index 000000000..2b7484b31 --- /dev/null +++ b/e2e-test/console_cli_readiness.py @@ -0,0 +1,95 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Continuous negative readiness evidence during the original DiskIO outage.""" + +import json +from pathlib import Path +import sys +import time + + +OUTAGE_SECONDS = 40 +OBSERVATION_SECONDS = 8 + + +def check_unavailable(values, endpoints): + assert type(endpoints) is int and endpoints > 0, endpoints + assert len(values) == endpoints, values + for value in values: + assert value.get("ready") is False and value.get("live") is True, value + observation = value.get("observation", {}) + generation = observation.get("generation") + checked = observation.get("checked_at_unix_ms") + age = observation.get("age_ms") + assert type(generation) is int and generation > 0, value + assert type(checked) is int and checked > 0, value + assert type(age) is int and age >= 0, value + assert observation.get("max_age_ms") == 3000, value + assert isinstance(value.get("details", {}).get("chunks"), str), value + assert value["details"]["chunks"], value + state = observation.get("state") + if state == "current": + assert age <= 3000 and value.get("chunks") == "unavailable", value + assert "chunks" in value.get("reasons", []), value + else: + # age_ms truncates Duration, so an expired check may display 3000. + assert state == "expired" and age >= 3000, value + assert value.get("chunks") == "unknown", value + assert "observation-expired" in value.get("reasons", []), value + + +def check_progress(history, endpoints): + assert history, "No outage observations were retained" + for sample in history: + check_unavailable(sample["responses"], endpoints) + for endpoint in range(endpoints): + previous = None + generations = set() + for sample in history: + observation = sample["responses"][endpoint]["observation"] + current = (observation["generation"], observation["checked_at_unix_ms"]) + if previous is not None: + assert current[0] >= previous[0], (endpoint, previous, current) + if current[0] == previous[0]: + assert current[1] == previous[1], (endpoint, previous, current) + else: + assert current[1] > previous[1], (endpoint, previous, current) + generations.add(current[0]) + previous = current + assert len(generations) >= 3, ("Sampler did not advance twice", endpoint, history) + + +def observe_unavailable(artifacts, snapshot, endpoints): + started = time.monotonic() + deadline = started + OUTAGE_SECONDS + time.sleep(OUTAGE_SECONDS - OBSERVATION_SECONDS) + history = [] + failure = None + try: + while True: + sample = {"started_at_unix_ms": time.time_ns() // 1_000_000, + "elapsed_seconds": time.monotonic() - started, + "responses": snapshot()} + sample["finished_at_unix_ms"] = time.time_ns() // 1_000_000 + history.append(sample) + check_unavailable(sample["responses"], endpoints) + remaining = deadline - time.monotonic() + if remaining <= 0: + break + time.sleep(min(0.25, remaining)) + check_progress(history, endpoints) + except BaseException as error: + failure = repr(error) + raise + finally: + report = {"outage_seconds": OUTAGE_SECONDS, + "observation_seconds": OBSERVATION_SECONDS, + "elapsed_seconds": time.monotonic() - started, + "failure": failure, "samples": history} + try: + (Path(artifacts) / "access-health-paused-history.json").write_text( + json.dumps(report, indent=2)) + except OSError as error: + if failure is None: + raise + print(f"Secondary readiness history write error: {error}", file=sys.stderr, flush=True) diff --git a/e2e-test/console_cli_recovery.py b/e2e-test/console_cli_recovery.py index 0d1b35c07..a384e682c 100644 --- a/e2e-test/console_cli_recovery.py +++ b/e2e-test/console_cli_recovery.py @@ -7,21 +7,16 @@ import time from fixture import request - - -def signal_diskio(cluster, identity, pid, signal): - expected = "/opt/crowdb/bin/crowdb-diskio" - check = f'test "$(readlink /proc/{pid}/exe)" = {expected} && kill -{signal} {pid}' - if signal == "CONT": - check = f"if test -e /proc/{pid}/exe; then {check}; fi" - cluster.command("exec", "--user", "crowdb", identity, "sh", "-c", check) +from console_cli_io_fault import DiskIoOutage +from console_cli_readiness import OUTAGE_SECONDS, observe_unavailable def health_snapshot(cluster, label): observed = [] for index, name in enumerate(cluster.servers): identity = cluster.inspect_owned(name)["Id"] - url = f"http://{cluster.addresses[index]}:9094/_crowdb/health/ready" + port = cluster.access_listeners(index)["health_port"] + url = f"http://{cluster.addresses[index]}:{port}/_crowdb/health/ready" try: value = cluster.command("exec", identity, "curl", "--silent", "--show-error", "--max-time", "3", url) @@ -32,27 +27,26 @@ def health_snapshot(cluster, label): return observed -def pause_and_write(cluster, pauses): +def block_and_write(cluster, outage): with (cluster.artifacts / "fault-s3.log").open("w") as output: clients = [] try: - for identity, pid in pauses: - signal_diskio(cluster, identity, pid, "STOP") - status = cluster.command("exec", identity, "cat", f"/proc/{pid}/status") - assert "State:\tT" in status, status - print("Paused three verified DiskIO processes across the 30-second I/O deadline", flush=True) - for number in range(3): - clients.append(subprocess.Popen(cluster.arguments("s3", "object", "put", - "writer-recovery", f"during-{number}", "--text", "during-fault"), - env=cluster.environment(), stdout=output, stderr=output)) - time.sleep(40) - unavailable = health_snapshot(cluster, "paused") - assert all(value.get("ready") is False for value in unavailable), unavailable - assert all(value.get("observation", {}).get("state") == "current" - for value in unavailable), unavailable + with outage: + outage.verify() + print("Blocked three owned DiskIO listeners across the 30-second I/O deadline", flush=True) + for number in range(3): + clients.append(subprocess.Popen(cluster.arguments("s3", "object", "put", + "writer-recovery", f"during-{number}", "--text", "during-fault"), + env=cluster.environment(), stdout=output, stderr=output)) + # Payload writes can fail before reaching the metadata journal. + clients.append(subprocess.Popen(cluster.arguments("s3", "bucket", "put", + "writer-recovery-metadata-fault"), env=cluster.environment(), + stdout=output, stderr=output)) + observe_unavailable(cluster.artifacts, + lambda: health_snapshot(cluster, "paused"), + len(cluster.servers)) + outage.verify() finally: - for identity, pid in pauses: - signal_diskio(cluster, identity, pid, "CONT") for client in clients: try: client.wait(timeout=60) @@ -95,8 +89,9 @@ def terminal_writer_recovery(cluster, cluster_id): assert type(pid) is int and pid > 1, service pauses.append((identities[index], pid)) (cluster.artifacts / "fault-injection.json").write_text(json.dumps({ - "diskio_processes": pauses, "pause_seconds": 40}, indent=2)) - pause_and_write(cluster, pauses) + "diskio_processes": pauses, "fault": "owned-namespace DiskIO packet drop", + "outage_seconds": OUTAGE_SECONDS}, indent=2)) + block_and_write(cluster, DiskIoOutage(cluster)) # No application traffic until independent health observation reports recovery. started = time.monotonic() try: diff --git a/e2e-test/console_cli_target.py b/e2e-test/console_cli_target.py index 4f24af1a5..5b0a4002f 100644 --- a/e2e-test/console_cli_target.py +++ b/e2e-test/console_cli_target.py @@ -3,6 +3,7 @@ """Attach a host CLI to ordinary Docker nodes without a launch receipt.""" import json +import time from pathlib import Path from allocation import create_network, node_data, start_node @@ -40,14 +41,15 @@ def start(self): f"CROWDB_CLUSTER={self.run_id}", "CROWDB_INITIAL_VOTERS=3", "CROWDB_DEPLOYMENT_MODE=local", f"CROWDB_PHYSICAL_HOST_ID={host}", - f"CROWDB_S3_PUBLIC_URI=http://{address}:9091", - f"CROWDB_ICEBERG_PUBLIC_URI=http://{address}:9092", - f"CROWDB_DATASET_PUBLIC_URI=http://{address}:9093", + f"CROWDB_S3_PUBLIC_URI=http://{address}:16000", + f"CROWDB_ICEBERG_PUBLIC_URI=http://{address}:17000", + f"CROWDB_DATASET_PUBLIC_URI=http://{address}:18000", "CROWDB_DISCOVERY_SEEDS=" + ",".join(f"http://{peer}:9095" for peer in peers)))) self.origins = [f"http://{address}:10000" for address in self.addresses] for name in self.servers: if not json.loads(self.command("inspect", name))[0]["State"]["Running"]: raise AssertionError("plain Docker node exited: " + self.command("logs", name)) + self.ready_deadline = time.monotonic() + 180 report = json.loads(self.invoke("cluster", "check", "--wait", "180", "--json", endpoint=self.endpoint(0))) assert report["phase"] == "ready", report @@ -62,12 +64,70 @@ def start(self): def endpoint(self, index): return f"http://{self.addresses[index]}:9090" + def access_listeners(self, index): + return {"s3_port": 16000, "health_port": 19000} + def inventory(self): containers = [json.loads(self.command("inspect", name))[0] for name in self.servers] - assert all(item["Config"]["Labels"].get(LABEL) == self.run_id for item in containers) + assert all(item["Config"]["Labels"].get(LABEL) == self.run_id + and item["Image"] == self.image and item["State"]["Running"] + for item in containers), "CLI inventory container ownership or image changed" + processes = {} + nodes = set() + for container in containers: + identity = container["Id"] + binding = json.loads(self.command("exec", identity, "cat", "/opt/crowdb/data/node-binding.json")) + node = binding["node_id"] + assert type(node) is int and node > 0 and node not in nodes, binding + assert binding["bootstrap"]["cluster_id"] == self.cluster_id, binding + nodes.add(node) + status = json.loads(self.command("exec", identity, "cat", "/opt/crowdb/run/status/monitor.json")) + for service, current in status["services"].items(): + assert type(current.get("pid")) is int and current["pid"] > 1, (node, service, "invalid monitor PID", current) + processes[node, service] = (current["pid"], current["generation"], current["operation_id"]) + self.command("exec", identity, "sh", "-c", """ + for pid do + state=$(awk '{print $3}' "/proc/$pid/stat") || exit 1 + case "$state" in ''|Z|X) + echo "owned service PID $pid is not live ($state)" >&2 + exit 1;; + esac + done + """, "owned-service-pids", *(str(current["pid"]) for current in status["services"].values())) + # Aggregate PID fields are optional fresh samples. Lifecycle invariance + # uses each owning monitor, including services absent from that cache. services = request(self.endpoint(0), "/api/servers") + assert len({item["id"] for item in services}) == len(services), services + kinds = {"paxos-kv", "diskdb", "diskio", "chunkdb", "chunk-kv", "access-server"} + for node in nodes: + registered = [item for item in services if item["node_id"] == node] + assert len(registered) == len(kinds) and {item["service_type"] for item in registered} == kinds, registered + required = {"web"} | {"kv" if item["service_type"] == "paxos-kv" else item["id"] for item in registered} + observed = {service for owner, service in processes if owner == node} + assert required <= observed, ("owned monitor service is absent", node, required, observed) + assert {item["node_id"] for item in services} == nodes, services + for item in services: + service = "kv" if item["service_type"] == "paxos-kv" else item["id"] + actual = processes[item["node_id"], service] + if item.get("pid") is not None: + assert item.get("pid_node_id") == item["node_id"], ("cached PID namespace changed", item) + assert item["pid"] == actual[0], ("cached PID differs from owning monitor", item, actual) + volatile = {"pid", "pid_node_id", "health", "start_pending"} + registrations = [{key: value for key, value in item.items() if key not in volatile} for item in services] return {"containers": [(item["Id"], item["Image"]) for item in containers], - "processes": sorted((item["id"], item["pid"]) for item in services)} + "processes": sorted((node, service, *identity) for (node, service), identity in processes.items()), + "registrations": sorted(registrations, key=lambda item: item["id"])} + + def check_ready(self, endpoint=None): + # Automatic catalog publication can temporarily fence an older serving + # grant. Use the original startup deadline for the final lifecycle gate; + # the operation does not receive another readiness budget or a replay. + remaining = int(self.ready_deadline - time.monotonic()) + assert remaining > 0, "CLI flow exhausted its original readiness deadline" + report = json.loads(self.invoke("cluster", "check", "--wait", str(remaining), "--json", + endpoint=endpoint)) + assert report["phase"] == "ready" and report["observation"]["cluster_id"] == self.cluster_id, report + return report def replay(self, checkpoint): if checkpoint != "attachment": @@ -77,9 +137,8 @@ def replay(self, checkpoint): "logical": console_cli_operations.logical, "data": console_cli_operations.data}[checkpoint] operation(self) + self.check_ready() assert self.inventory() == before, "logical CLI operation changed runtime lifecycle" - report = json.loads(self.invoke("cluster", "check", "--json")) - assert report["phase"] == "ready" and report["observation"]["cluster_id"] == self.cluster_id (self.artifacts / "operation-proof.json").write_text(json.dumps({ "cluster_id": self.cluster_id, "operation": checkpoint, "inventory": before}, indent=2)) print("PASS selected-cluster CLI", checkpoint, flush=True) @@ -107,8 +166,7 @@ def attachment(self): assert self.invoke("s3", "object", "get", "cli-attach", "witness", endpoint=self.endpoint(index)) == value self.invoke("cluster", "stop", endpoint=self.endpoint(0), expected_exit=2) - report = json.loads(self.invoke("cluster", "check", "--json", endpoint=self.endpoint(2))) - assert report["phase"] == "ready" and report["observation"]["cluster_id"] == self.cluster_id + self.check_ready(endpoint=self.endpoint(2)) assert self.inventory() == before, "attachment changed container or service lifecycle" assert not (self.root / "persistent/oci-clusters").exists(), "attachment created a runtime receipt" (self.artifacts / "attachment-proof.json").write_text(json.dumps({ diff --git a/e2e-test/console_native_fixture.py b/e2e-test/console_native_fixture.py index abed7d913..4581716de 100644 --- a/e2e-test/console_native_fixture.py +++ b/e2e-test/console_native_fixture.py @@ -3,6 +3,7 @@ """Production-policy release nodes with explicit admission and fixed ports.""" import json +import re import urllib.request import urllib.error from urllib.parse import urlparse @@ -268,6 +269,14 @@ def control(self, body): ), config="native.config.ts", expected=4, extra_env={"CROWDB_NATIVE_CHUNK_FIXTURE": json.dumps(chunks)}) return {"passed": True} + if self.case_name == "console::native_browser::journal_identity_and_owner_recovery": + bucket = body.get("journal_bucket") + if not isinstance(bucket, str) or not re.fullmatch(r"journal-[a-f0-9]{32}", bucket): + raise ValueError("journal diagnostics require their settled owned object") + self.browser_timeout = 180 + self.browser("native diagnostics: journal identity survives Chunk navigation and owner interruption", + config="native.config.ts", extra_env={"CROWDB_NATIVE_JOURNAL_BUCKET": bucket}) + return {"passed": True} cases = { "console::native_browser::catalog_resource_navigation": ( "native diagnostics: Iceberg empty Catalog omits the first-success guide", diff --git a/e2e-test/console_native_policy.py b/e2e-test/console_native_policy.py index d0ed9e1ac..d9f2320d2 100644 --- a/e2e-test/console_native_policy.py +++ b/e2e-test/console_native_policy.py @@ -3,13 +3,12 @@ """Accept the owned diskless policy before attaching unadmitted fixture media.""" from allocation import LABEL +from profiles.setup_choices import diskless_policy def accept(cluster, index, data, physical_host): policy = cluster.artifacts / f"node-policy-{index}.toml" - kinds = ("paxos-kv", "diskdb", "diskio", "chunkdb", "chunk-kv", "access-server") - services = ", ".join(f'{{ kind = "{kind}", enabled = false }}' for kind in kinds) - policy.write_text(f'services = [{services}]\n[storage]\nmode = "diskless"\n') + policy.write_text(diskless_policy()) environment = ("CROWDB_DEPLOYMENT_MODE=production", "CROWDB_INITIAL_VOTERS=3", f"CROWDB_CLUSTER={cluster.run_id}", "CROWDB_SSH_PASSWORD=crowdb", f"CROWDB_PHYSICAL_HOST_ID={physical_host}", diff --git a/e2e-test/fixture.py b/e2e-test/fixture.py index a6107a440..11ee2aa50 100644 --- a/e2e-test/fixture.py +++ b/e2e-test/fixture.py @@ -16,6 +16,7 @@ from client import run_client from kv_node import arguments as kv_arguments from control import Control +from fixture_result import CaseNotExecuted from kv_topology import ready_nodes, wire_nodes HTTP = urllib.request.build_opener(urllib.request.ProxyHandler({})) @@ -250,26 +251,49 @@ def __enter__(self): def __exit__(self, _kind, error, _traceback): from failure_logs import print_failure + not_executed = isinstance(error, CaseNotExecuted) + if not_executed: + self.timings["not_executed_reason"] = redact(str(error)) if self.closed: - self.timings["status"] = "pass" if error is None else "failed" + self.timings["status"] = ("not_executed" if not_executed + else "pass" if error is None else "failed") self.write_timings() return + if not_executed: + print(redact(f"[NOT EXECUTED CASE] {getattr(self, 'case_name', self.run_id)}: " + f"{error}\nArtifacts: {self.artifacts}"), flush=True) collection_error = None try: self.diagnostics() except (OSError, ValueError, RuntimeError, subprocess.SubprocessError) as diagnostic_error: collection_error = diagnostic_error - if error is not None: + if error is not None and not not_executed: error.add_note(str(diagnostic_error)) - if error is not None or collection_error is not None: + reported_error = collection_error if not_executed else error or collection_error + if reported_error is not None: print_failure(self.artifacts, self.data_paths, getattr(self, "case_name", self.run_id), - error if error is not None else collection_error, redact) + reported_error, redact) try: - self.timings["status"] = "pass" if error is None and collection_error is None else "failed" + self.timings["status"] = ("failed" if collection_error is not None + else "not_executed" if not_executed + else "pass" if error is None else "failed") self.close() except (OSError, ValueError, RuntimeError, subprocess.SubprocessError) as cleanup_error: + if not_executed: + self.timings["status"] = "failed" + self.write_timings() + print_failure(self.artifacts, self.data_paths, getattr(self, "case_name", self.run_id), + cleanup_error, redact) + if collection_error is not None: + collection_error.add_note(str(cleanup_error)) + raise collection_error from cleanup_error + raise if error is None: + print_failure(self.artifacts, self.data_paths, getattr(self, "case_name", self.run_id), + cleanup_error, redact) raise error.add_note(str(cleanup_error)) - if error is None and collection_error is not None: + print(redact(f"[CLEANUP ERROR] {getattr(self, 'case_name', self.run_id)}: " + f"{cleanup_error}\nArtifacts: {self.artifacts}"), flush=True) + if (error is None or not_executed) and collection_error is not None: raise collection_error diff --git a/e2e-test/fixture_result.py b/e2e-test/fixture_result.py new file mode 100644 index 000000000..55c51686a --- /dev/null +++ b/e2e-test/fixture_result.py @@ -0,0 +1,10 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Distinguish an undispatched client from a failed fixture operation.""" + + +class CaseNotExecuted(RuntimeError): + """An entered fixture whose client was blocked by another startup.""" + + def __init__(self, blocked_by): + super().__init__(f'browser client not executed: batch startup failed in {blocked_by}') diff --git a/e2e-test/oci_reference_recovery.py b/e2e-test/oci_reference_recovery.py index 604ad5a8d..537b8f65a 100644 --- a/e2e-test/oci_reference_recovery.py +++ b/e2e-test/oci_reference_recovery.py @@ -12,6 +12,7 @@ from fixture import HTTP, LABEL, request, wait from profiles.node import request as admission_request +from profiles.setup_choices import diskless_policy import oci_reference_runtime as runtime import oci_reference_protocols as protocols @@ -24,11 +25,8 @@ def create_nonvoter(cluster): name = cluster.expected_names[3] volume = runtime.node_volume(cluster, 3) policy = cluster.artifacts / 'later-member.toml' - lines = ['version = 1', 'mode = "local"', f'cluster = "{cluster.run_id}"', - 'initial_voters = 3', '[storage]', 'mode = "diskless"'] - for kind in ('paxos-kv', 'diskdb', 'diskio', 'chunkdb', 'chunk-kv', 'access-server'): - lines += ['[[services]]', f'kind = "{kind}"', 'enabled = false'] - content = '\n'.join(lines) + '\n' + content = ('version = 1\nmode = "local"\n' + f'cluster = "{cluster.run_id}"\ninitial_voters = 3\n' + diskless_policy()) if policy.exists(): assert policy.read_text() == content else: @@ -131,6 +129,15 @@ def replace_addresses(cluster, before): def verify(cluster, cluster_id, witness, credentials): assert cluster.implementation == 'docker' and cluster.node_count == 3 + # This case submits one explicit later-member admission. Stop automatic + # admission before exposing that candidate, rather than racing two callers. + for index in range(3): + disabled = admission_request(origin(cluster, index), '/api/node/auto-join', + {'enabled': False}) + assert disabled['enabled'] is False + wait(lambda index=index: request(origin(cluster, index), '/api/node/auto-join'), + lambda value: value['observation']['phase'] == 'disabled', + 'automatic admission quiesced before manual admission', seconds=10) cluster.expected_names.append(cluster.run_id + '-node4') create_nonvoter(cluster) cluster.command('start', cluster.expected_names[3]) diff --git a/e2e-test/oci_reference_runtime.py b/e2e-test/oci_reference_runtime.py index 1c024232b..5030c7bc4 100644 --- a/e2e-test/oci_reference_runtime.py +++ b/e2e-test/oci_reference_runtime.py @@ -90,7 +90,7 @@ def create_docker_nodes(cluster, indexes): "--env", f"CROWDB_CLUSTER={cluster.run_id}", "--env", f"CROWDB_INITIAL_VOTERS={cluster.node_count}"] for protocol, port in ports(cluster.port_base + index * 10).items(): - internal = 9090 + (port - cluster.port_base - index * 10) + internal = {"UI": 9090, "S3": 16000, "ICEBERG": 17000, "DATASET": 18000}[protocol] arguments += ["--publish", f"127.0.0.1:{port}:{internal}"] if protocol != "UI": arguments += ["--env", f"CROWDB_{protocol}_PUBLIC_URI=http://127.0.0.1:{port}"] diff --git a/e2e-test/oci_shortcuts.py b/e2e-test/oci_shortcuts.py index 21de03cfe..b495f4aa3 100644 --- a/e2e-test/oci_shortcuts.py +++ b/e2e-test/oci_shortcuts.py @@ -103,7 +103,11 @@ def credentials(offset): retained_credentials=True, retained_management_write=True, unrelated_container_ids=unrelated, actions=actions) finally: - invoke('oci-clean', '--all', '--delete-data' if success else '--keep-data') + for name in (one, other, three): + receipt = root / 'persistent/oci-clusters' / (name + '.json') + if receipt.exists(): + invoke('oci-clean', '--cluster', name, + '--delete-data' if success else '--keep-data') assert all(not container_ids(name) for name in (one,other,three,missing)) (cluster.artifacts/'shortcuts-proof.json').write_text(json.dumps(proof,indent=2)) print('PASS real Pixi one/three-node shortcuts, retained data and independent scoped cleanup without CLI',flush=True) diff --git a/e2e-test/profiles/bridge_node.py b/e2e-test/profiles/bridge_node.py index 7d1a6f317..e767b250e 100644 --- a/e2e-test/profiles/bridge_node.py +++ b/e2e-test/profiles/bridge_node.py @@ -11,6 +11,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from admission_profile import cleanup as cleanup_runtime +from profiles.setup_choices import diskless_policy IMAGE = os.environ["CROWDB_CONTAINER_IMAGE"] PREFIX = os.environ["CROWDB_E2E_PROFILE_NAME"] @@ -32,12 +33,10 @@ def start_node(name, *, default_setup=False): data = DATA / name.removeprefix(PREFIX + "-") data.mkdir(mode=0o755) policy = DATA / (name.removeprefix(PREFIX + "-") + ("-default-policy.toml" if default_setup else "-policy.toml")) - kinds = ("paxos-kv", "diskdb", "diskio", "chunkdb", "chunk-kv", "access-server") - services = ", ".join(f'{{ kind = "{kind}", enabled = false }}' for kind in kinds) if default_setup: policy.write_text('[storage]\nmode = "automatic"\ncapacity_bytes = 17179869184\nreserve_bytes = 1073741824\nsparse = false\n') else: - policy.write_text(f'services = [{services}]\n[storage]\nmode = "diskless"\n') + policy.write_text(diskless_policy()) docker("run", "--rm", "--name", PREFIX + "-prepare", "--network", "none", "--label", f"org.crowdb.e2e.run={PREFIX}", "--user", "0", "--cpu-shares", "1024", "--env", "TOKIO_WORKER_THREADS=1", "--memory", "32m", "--entrypoint", "/bin/chown", diff --git a/e2e-test/profiles/setup_choices.py b/e2e-test/profiles/setup_choices.py index 96eac2aa5..eaa7f504f 100644 --- a/e2e-test/profiles/setup_choices.py +++ b/e2e-test/profiles/setup_choices.py @@ -4,6 +4,21 @@ def diskless_setup(): - return {"storage": {"mode": "diskless"}, "services": [ + services = [ + {"kind": "paxos-kv", "enabled": True, + "listeners": {"http_port": 10000, "rpc_port": 10100}}] + services += [ {"kind": kind, "enabled": False, "listeners": {}} - for kind in ("paxos-kv", "diskdb", "diskio", "chunkdb", "chunk-kv", "access-server")]} + for kind in ("diskdb", "diskio", "chunkdb", "chunk-kv", "access-server")] + return {"storage": {"mode": "diskless"}, "services": services} + + +def diskless_policy(): + lines = ['[storage]', 'mode = "diskless"'] + for service in diskless_setup()["services"]: + lines += ['[[services]]', f'kind = "{service["kind"]}"', + f'enabled = {str(service["enabled"]).lower()}'] + if service["listeners"]: + lines += ['[services.listeners]', + *(f'{key} = {value}' for key, value in service["listeners"].items())] + return '\n'.join(lines) + '\n' diff --git a/e2e-test/report.py b/e2e-test/report.py index da132ec0f..5ca273f81 100644 --- a/e2e-test/report.py +++ b/e2e-test/report.py @@ -17,8 +17,10 @@ def write_report(root, timings): phase.get("exit_code") == phase.get("expected_exit_code", 0) for phase in phases) actions = [json.loads(action.read_text()) for action in sorted( (directory / "control").glob("*.response.json"))] + status = ("not_executed" if fixture.get("status") == "not_executed" + else "pass" if completed else "failed") cases.append({**case, "actions": actions, "artifacts": str(directory), "phases": phases, - "fixture": fixture, "status": "pass" if completed else "failed"}) + "fixture": fixture, "status": status}) cases.sort(key=lambda case: case["fixture"].get("total", 0), reverse=True) slowest_clients = sorted( ({"case": case["case"], "scope": case["scope"], **phase} diff --git a/e2e-test/scenarios.py b/e2e-test/scenarios.py index f772710c9..19edccc1b 100644 --- a/e2e-test/scenarios.py +++ b/e2e-test/scenarios.py @@ -158,7 +158,7 @@ class Case: Case("kv.recovery", "kv::snapshot::snapshot_import_precedes_wiring_and_catches_up_tail", (0, 1, 2), groups_per_node=((1,), (1,), ())), Case("kv.membership", "kv::reconfiguration::quorum_expands_and_shrinks_without_losing_writes", - (0, 1, 2, 3, 4)), + (0, 1, 2, 3, 4), groups_per_node=((1,), (1,), (1,), (), ())), Case("kv.routing", "kv::cluster::topology_and_follower_hint", (10, 20, 30), True), Case("kv.routing", "kv::cluster::dynamic_groups_isolate_data", (0, 1, 2), True), Case("kv.management", "kv::management::management_lifecycle"), @@ -846,6 +846,10 @@ def execute(case, owned=None): cluster.crash_restart() cluster.client(bundle, "verify", case.name) else: + if case.scope == "console.native.browser.dataset": + # Production placement and browser execution each retain the + # case's bounded client budget in the same owned fixture. + cluster.client(bundle, "prepare", case.name) cluster.client(bundle, case.name.rsplit("::", 1)[-1], case.name) def stable_network(case): diff --git a/e2e-test/tests/access_iceberg/file_http/concurrency.rs b/e2e-test/tests/access_iceberg/file_http/concurrency.rs index 67b159513..5a0ff9fc4 100644 --- a/e2e-test/tests/access_iceberg/file_http/concurrency.rs +++ b/e2e-test/tests/access_iceberg/file_http/concurrency.rs @@ -1,32 +1,73 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +use super::super::signed_file::TestFileClient; use super::setup::{path, setup}; -use reqwest::Method; +use reqwest::{Method, StatusCode}; +use std::time::Duration; + +const MAX_SLOWDOWN_ATTEMPTS: usize = 4; +const SLOWDOWN_BASE_DELAY_MS: u64 = 100; +const SLOWDOWN_STAGGER_MS: u64 = 25; + +async fn put_with_slowdown_retry( + client: TestFileClient, + object: String, + payload: Vec, + index: usize, +) -> (StatusCode, String, usize) { + for attempt in 1..=MAX_SLOWDOWN_ATTEMPTS { + let response = client.send(Method::PUT, &object, "", &payload, false).await; + let status = response.status(); + let body = response.text().await.unwrap(); + if status != StatusCode::SERVICE_UNAVAILABLE + || !body.contains("SlowDown") + || attempt == MAX_SLOWDOWN_ATTEMPTS + { + return (status, body, attempt); + } + + let delay_ms = + SLOWDOWN_BASE_DELAY_MS * (1 << (attempt - 1)) + (index as u64 % 8) * SLOWDOWN_STAGGER_MS; + eprintln!("upload {index}: transient SlowDown on attempt {attempt}; retrying in {delay_ms}ms"); + tokio::time::sleep(Duration::from_millis(delay_ms)).await; + } + + unreachable!("the retry loop always returns on its final attempt") +} + #[tokio::test(flavor = "multi_thread", worker_threads = 8)] #[ignore = "requires the owned release Iceberg FileIO stack"] async fn thirty_two_concurrent_direct_puts() { let (_stack, _process, client, table) = setup().await; let payload = vec![0x5a; 8 * 1024 * 1024]; + // Prime one large writer before releasing the synchronized burst. The + // first foreground allocation also settles the ChunkDB/DiskIO route after + // the listener becomes reachable, so the burst measures concurrent writes + // instead of startup discovery. + let warmup = path(table, "data/parallel-warmup.parquet"); + let response = client.send(Method::PUT, &warmup, "", &payload, false).await; + assert_eq!(response.status(), 200, "{}", response.text().await.unwrap()); let mut requests = tokio::task::JoinSet::new(); let barrier = std::sync::Arc::new(tokio::sync::Barrier::new(33)); for index in 0..32 { let object = path(table, &format!("data/parallel-{index}.parquet")); - let request = client.request(Method::PUT, &object, "", &payload, false, None); + let payload = payload.clone(); + let client = client.clone(); let barrier = barrier.clone(); requests.spawn(async move { barrier.wait().await; - let response = request.send().await.unwrap(); - (index, response.status(), response.text().await.unwrap()) + let (status, body, attempts) = put_with_slowdown_retry(client, object, payload, index).await; + (index, status, body, attempts) }); } barrier.wait().await; let mut failures = Vec::new(); while let Some(result) = requests.join_next().await { - let (index, status, body) = result.unwrap(); - println!("upload {index}: {status} {body}"); - if status != 200 { - failures.push((index, status, body)); + let (index, status, body, attempts) = result.unwrap(); + println!("upload {index}: {status} attempts={attempts} {body}"); + if status != StatusCode::OK { + failures.push((index, status, attempts, body)); } } assert!(failures.is_empty(), "failed uploads: {failures:?}"); diff --git a/e2e-test/tests/access_iceberg/gc/convergence.rs b/e2e-test/tests/access_iceberg/gc/convergence.rs index 13a3457f8..273ad0e72 100644 --- a/e2e-test/tests/access_iceberg/gc/convergence.rs +++ b/e2e-test/tests/access_iceberg/gc/convergence.rs @@ -36,26 +36,36 @@ struct TestGcConvergence { async fn expired_inline_backlog_completes_with_foreground_progress() { let stack = TestNativeStorage::new(); let (store, context, table) = seed_table(&stack).await; - let backlog = seed_expired_backlog(store.as_ref(), context, table, 128).await; + let started = Instant::now(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(3300); + let pressure = TestGcPressure::prepare_before(deadline).await.unwrap(); + let backlog = tokio::time::timeout_at( + deadline, + seed_expired_backlog(store.as_ref(), context, table, 128), + ) + .await + .expect("inline GC backlog seed exceeded the existing case deadline"); assert_eq!(backlog.files.len(), 128); assert!(backlog.files.len() > usize::from(GcLimits::default().page_items)); assert!(backlog.dropped_ms + GcTask::PURGE_DELAY_MS < now_ms()); - let server = TestFrontend::start(12).await; let mut check = TestGcConvergence { store, context, table, backlog, - started: Instant::now(), + started, task_gets: 0, verification_gets: 0, observations: Vec::new(), last: None, - pressure: None, + pressure: Some(pressure), }; check.report("seeded"); - let result = - tokio::time::timeout(Duration::from_secs(3300), check.run(&server.origin, &stack.seeds)).await; + let result = tokio::time::timeout_at(deadline, async { + TestFrontend::start(12).await; + check.run(&stack.seeds).await + }) + .await; let result = result.unwrap_or_else(|_| Err("inline GC convergence exceeded 3300 seconds".into())); if let Err(error) = result { check.report("failed"); @@ -68,15 +78,15 @@ async fn expired_inline_backlog_completes_with_foreground_progress() { if let Some(pressure) = &mut check.pressure { pressure.stop().await; } - server.stop().await; + crate::common::control::node("stop", 12).await; panic!("inline GC convergence failed: {error}"); } check.report("complete"); - server.stop().await; + crate::common::control::node("stop", 12).await; } impl TestGcConvergence { - async fn run(&mut self, origin: &str, seeds: &[String]) -> Result<(), String> { + async fn run(&mut self, seeds: &[String]) -> Result<(), String> { let identity = OperationId::from_bytes(self.table.as_bytes()).map_err(|error| error.to_string())?; let repository = GcRepository::new(self.store.clone()); loop { @@ -87,8 +97,14 @@ impl TestGcConvergence { .map_err(|error| error.to_string())? { self.observe(&task)?; - if let Some(pressure) = &mut self.pressure { - pressure.observe(task.deleted).await?; + let pressure = self + .pressure + .as_mut() + .ok_or("prepared foreground pressure is missing")?; + if pressure.released() { + pressure + .observe(task.deleted, task.phase == GcPhase::Complete) + .await?; } else if task.deleted > 0 { if task.phase == GcPhase::Complete { return Err("GC completed before the foreground overlap could be started".into()); @@ -100,7 +116,7 @@ impl TestGcConvergence { "revision":task.revision,"phase":format!("{:?}",task.phase), }) ); - self.pressure = Some(TestGcPressure::start(origin, seeds.to_vec())?); + pressure.release(seeds.to_vec())?; } if task.phase == GcPhase::Complete && self.pressure.as_ref().is_some_and(TestGcPressure::complete) diff --git a/e2e-test/tests/access_iceberg/gc/convergence/pressure.rs b/e2e-test/tests/access_iceberg/gc/convergence/pressure.rs index a29500daa..543f4afb3 100644 --- a/e2e-test/tests/access_iceberg/gc/convergence/pressure.rs +++ b/e2e-test/tests/access_iceberg/gc/convergence/pressure.rs @@ -1,10 +1,13 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::super::foreground::{write_s3_with_delay, SDK_PRESSURE_SCRIPT}; -use crowdb_access_iceberg::key::OperationId; +mod sdk; + +use super::super::foreground::write_s3_with_delay; +use crate::{access_iceberg::frontend::TestFrontend, common::control}; +use futures::FutureExt; +use sdk::TestSdkWorker; use std::{ - path::PathBuf, sync::{ atomic::{AtomicUsize, Ordering}, Arc, @@ -13,8 +16,7 @@ use std::{ }; pub(in crate::access_iceberg::gc) struct TestGcPressure { - sdk: std::process::Child, - stdout: PathBuf, + sdk: TestSdkWorker, writes: Option>, progress: Arc, previous_active: Option<(u64, u64)>, @@ -23,40 +25,76 @@ pub(in crate::access_iceberg::gc) struct TestGcPressure { } impl TestGcPressure { - pub(in crate::access_iceberg::gc) fn start(origin: &str, seeds: Vec) -> Result { - let stdout = PathBuf::from(std::env::var("CROWDB_E2E_ARTIFACTS").map_err(|error| error.to_string())?) - .join(format!("gc-inline-sdk-{}.log", OperationId::random())); - let output = std::fs::File::create(&stdout).map_err(|error| error.to_string())?; - let sdk = std::process::Command::new("pixi") - .args(["run", "-e", "iceberg-e2e", "python", "-c", SDK_PRESSURE_SCRIPT]) - .args([origin, "12", "0.5", "gc-convergence"]) - .stdout(output) - .spawn() - .map_err(|error| error.to_string())?; - let progress = Arc::new(AtomicUsize::new(0)); - let writes = tokio::spawn(write_s3_with_delay( - seeds, - progress.clone(), - Duration::from_millis(250), - )); + pub(in crate::access_iceberg::gc) async fn prepare_before( + deadline: tokio::time::Instant, + ) -> Result { + // Call before publishing an executable purge marker. This ordinary + // frontend warms the real SDK while no seeded deletion can race it. + let result = tokio::time::timeout_at(deadline, async { + let server = TestFrontend::start(12).await; + let mut sdk = TestSdkWorker::prepare(&server.origin)?; + sdk.wait_ready().await?; + // Restart after publishing the marker so its first scan sees the + // backlog, without waiting for the production idle scan interval. + server.stop().await; + Ok::<_, String>(sdk) + }) + .await; + let sdk = match result + .unwrap_or_else(|_| Err("SDK warmup exceeded the existing GC case deadline".into())) + { + Ok(sdk) => sdk, + Err(error) => { + let cleanup = std::panic::AssertUnwindSafe(control::node("stop", 12)) + .catch_unwind() + .await; + if cleanup.is_err() { + eprintln!("SDK warmup frontend cleanup also failed; retaining the original warmup error"); + } + return Err(error); + } + }; Ok(Self { sdk, - stdout, - writes: Some(writes), - progress, + writes: None, + progress: Arc::new(AtomicUsize::new(0)), previous_active: None, deletion_intervals: Vec::new(), sdk_complete: false, }) } - pub(in crate::access_iceberg::gc) async fn observe(&mut self, deleted: u64) -> Result<(), String> { - if let Some(status) = self.sdk.try_wait().map_err(|error| error.to_string())? { + pub(in crate::access_iceberg::gc) fn released(&self) -> bool { + self.sdk.released() + } + + pub(in crate::access_iceberg::gc) fn release(&mut self, seeds: Vec) -> Result<(), String> { + self.sdk.release()?; + self.writes = Some(tokio::spawn(write_s3_with_delay( + seeds, + self.progress.clone(), + Duration::from_millis(250), + ))); + Ok(()) + } + + pub(in crate::access_iceberg::gc) async fn observe( + &mut self, + deleted: u64, + gc_complete: bool, + ) -> Result<(), String> { + if !self.released() { + return Err("foreground observation requires the deletion-boundary release".into()); + } + if let Some(status) = self.sdk.try_wait()? { if !status.success() { return Err(format!("official SDK foreground failed: {status}")); } self.sdk_complete = true; } + if gc_complete && self.writes.is_none() { + self.sdk.stop()?; + } if self .writes .as_ref() @@ -86,30 +124,31 @@ impl TestGcPressure { } pub(in crate::access_iceberg::gc) fn complete(&self) -> bool { - self.sdk_complete && self.writes.is_none() + self.released() && self.sdk_complete && self.writes.is_none() } pub(in crate::access_iceberg::gc) async fn stop(&mut self) { + if let Err(error) = self.sdk.stop() { + eprintln!("SDK cleanup also failed: {error}"); + } if let Some(writes) = self.writes.take() { writes.abort(); let _ = writes.await; } - if !self.sdk_complete { - let _ = self.sdk.kill(); - let _ = self.sdk.wait(); + if let Err(error) = self.sdk.terminate() { + eprintln!("SDK cleanup also failed: {error}"); } - match std::fs::read_to_string(&self.stdout) { - Ok(output) => print!("{output}"), - Err(error) => eprintln!("SDK stdout artifact {}: {error}", self.stdout.display()), + if let Err(error) = self.sdk.output() { + eprintln!("SDK output: {error}"); } } - pub(in crate::access_iceberg::gc) fn finish(self) -> Result<(), String> { + pub(in crate::access_iceberg::gc) fn finish(mut self) -> Result<(), String> { if !self.complete() { return Err("foreground workloads did not finish".into()); } - let stdout = std::fs::read_to_string(&self.stdout).map_err(|error| error.to_string())?; - print!("{stdout}"); + self.sdk.terminate()?; + let stdout = self.sdk.output()?; if self.deletion_intervals.is_empty() { return Err("no actual deletion advanced while both foreground workloads were active".into()); } @@ -118,6 +157,23 @@ impl TestGcPressure { .find_map(|line| line.strip_prefix("gc_foreground_official_sdk ")) .ok_or("official SDK latency report is missing")?; let report: serde_json::Value = serde_json::from_str(report).map_err(|error| error.to_string())?; + for operation in [ + "create_table", + "commit", + "load_table", + "credentials", + "read_metadata", + "drop_table", + ] { + if report[operation]["count"] + .as_u64() + .map_or(true, |count| count < 48) + { + return Err(format!( + "official SDK did not complete its minimum foreground workload: {operation}" + )); + } + } let overlapping = report .as_object() .ok_or("SDK report must be an object")? @@ -143,12 +199,20 @@ impl TestGcPressure { "gc_foreground_deletion_overlap {}", serde_json::json!({ "deletion_intervals_unix_ns":self.deletion_intervals,"small_writes":64, - "sdk_workers":4,"tables_per_worker":12,"inter_table_sleep_ms":500, + "sdk_workers":4,"minimum_tables_per_worker":12,"inter_table_sleep_ms":500, "small_write_interval_ms":250,"sleeps_in_latency_samples":false, - "sdk_stdout_artifact":self.stdout, - "scope":"independent fixed load; latency is not directly comparable to the original three-table, 50-ms probe", + "sdk_stdout_artifact":self.sdk.stdout(), + "scope":"continuous foreground through cleanup and small-writer completion; latency is not a fixed-load baseline", }) ); Ok(()) } } + +impl Drop for TestGcPressure { + fn drop(&mut self) { + if let Some(writes) = self.writes.take() { + writes.abort(); + } + } +} diff --git a/e2e-test/tests/access_iceberg/gc/convergence/pressure/sdk.rs b/e2e-test/tests/access_iceberg/gc/convergence/pressure/sdk.rs new file mode 100644 index 000000000..85243af52 --- /dev/null +++ b/e2e-test/tests/access_iceberg/gc/convergence/pressure/sdk.rs @@ -0,0 +1,234 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Own the real SDK process across warmup, endpoint restarts and release. + +use super::super::super::foreground::SDK_PRESSURE_SCRIPT; +use crowdb_access_iceberg::key::OperationId; +use std::os::unix::process::CommandExt; +use std::{ + path::PathBuf, + process::{Child, ExitStatus}, + time::{Duration, Instant}, +}; + +pub(super) struct TestSdkWorker { + child: Child, + stdout: PathBuf, + stderr: PathBuf, + control: PathBuf, + released: bool, + reported: bool, + cleanup: Option>, +} + +impl TestSdkWorker { + pub(super) fn prepare(origin: &str) -> Result { + let stdout = PathBuf::from(std::env::var("CROWDB_E2E_ARTIFACTS").map_err(|error| error.to_string())?) + .join(format!("gc-inline-sdk-{}.log", OperationId::random())); + let stderr = stdout.with_extension("stderr.log"); + let control = stdout.with_extension("control"); + std::fs::create_dir(&control).map_err(|error| error.to_string())?; + let output = std::fs::File::create(&stdout).map_err(|error| error.to_string())?; + let errors = std::fs::File::create(&stderr).map_err(|error| error.to_string())?; + let mut command = std::process::Command::new("pixi"); + command + .args(["run", "-e", "iceberg-e2e", "python", "-c", SDK_PRESSURE_SCRIPT]) + .args([origin, "12", "0.5", "gc-convergence"]) + .arg(&control) + .stdout(output) + .stderr(errors) + .process_group(0); + let child = command.spawn().map_err(|error| error.to_string())?; + Ok(Self { + child, + stdout, + stderr, + control, + released: false, + reported: false, + cleanup: None, + }) + } + + pub(super) async fn wait_ready(&mut self) -> Result<(), String> { + loop { + if self.ready()? { + println!( + "gc_foreground_ready {}", + serde_json::json!({"workers":4,"run_exists":false,"stdout":self.stdout,"stderr":self.stderr}) + ); + return Ok(()); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + } + + fn ready(&mut self) -> Result { + if let Some(status) = self.try_wait()? { + return Err(format!("official SDK exited before release: {status}")); + } + if self + .control + .join("run") + .try_exists() + .map_err(|error| error.to_string())? + { + return Err("official SDK was released before warmup completed".into()); + } + for worker in 0..4 { + if !self + .control + .join(format!("ready-{worker}")) + .try_exists() + .map_err(|error| error.to_string())? + { + return Ok(false); + } + } + Ok(true) + } + + fn check_failure(&self) -> Result<(), String> { + for worker in 0..4 { + let failed = self.control.join(format!("failed-{worker}")); + if failed.try_exists().map_err(|error| error.to_string())? { + return Err(format!( + "SDK worker {worker} failed: {}", + std::fs::read_to_string(failed).map_err(|error| error.to_string())? + )); + } + } + Ok(()) + } + + pub(super) fn release(&mut self) -> Result<(), String> { + if self.released || !self.ready()? { + return Err("official SDK release requires four parked ready workers".into()); + } + std::fs::write(self.control.join("run"), b"").map_err(|error| error.to_string())?; + self.released = true; + Ok(()) + } + + pub(super) fn released(&self) -> bool { + self.released + } + + pub(super) fn try_wait(&mut self) -> Result, String> { + self.check_failure()?; + self.child.try_wait().map_err(|error| error.to_string()) + } + + pub(super) fn stop(&self) -> Result<(), String> { + std::fs::write(self.control.join("stop"), b"") + .map_err(|error| format!("SDK stop signal {}: {error}", self.control.display())) + } + + pub(super) fn stdout(&self) -> &std::path::Path { + &self.stdout + } + + pub(super) fn output(&mut self) -> Result { + let stdout = std::fs::read_to_string(&self.stdout).map_err(|error| error.to_string())?; + if !self.reported { + print!("{stdout}"); + match std::fs::read_to_string(&self.stderr) { + Ok(stderr) => eprint!("{stderr}"), + Err(error) => eprintln!("SDK stderr artifact {}: {error}", self.stderr.display()), + } + self.reported = true; + } + Ok(stdout) + } + + pub(super) fn terminate(&mut self) -> Result<(), String> { + if let Some(result) = &self.cleanup { + return result.clone(); + } + let mut errors = self.stop().err().into_iter().collect::>(); + let started = Instant::now(); + let deadline = started + Duration::from_secs(2); + match self.wait_until(started + Duration::from_millis(1500)) { + Ok(true) => {} + result => { + if let Err(error) = result { + errors.push(error); + } + // A reaped launcher does not imply that its Python descendants + // exited. Kill only the new group created by this spawn. + if let Err(error) = self.signal_group("-KILL") { + errors.push(error); + } + match self.wait_until(deadline) { + Ok(true) => {} + Ok(false) => errors.push(format!( + "SDK owned group {} did not exit and reap within cleanup's two seconds", + self.child.id() + )), + Err(error) => errors.push(error), + } + } + } + let result = if errors.is_empty() { + Ok(()) + } else { + Err(errors.join("\n")) + }; + self.cleanup = Some(result.clone()); + result + } + + fn wait_until(&mut self, deadline: Instant) -> Result { + loop { + let reaped = self + .child + .try_wait() + .map_err(|error| format!("SDK child status: {error}"))? + .is_some(); + if !self.signal_group("-0")? && reaped { + // try_wait has cached the exit status, so wait cannot block. + self.child + .wait() + .map_err(|error| format!("SDK child wait: {error}"))?; + return Ok(true); + } + if Instant::now() >= deadline { + return Ok(false); + } + std::thread::sleep(Duration::from_millis(10)); + } + } + + fn signal_group(&self, signal: &str) -> Result { + let output = std::process::Command::new("/bin/kill") + .env("LC_ALL", "C") + .args([signal, "--", &format!("-{}", self.child.id())]) + .output() + .map_err(|error| format!("SDK owned group {} {signal}: {error}", self.child.id()))?; + if output.status.success() { + Ok(true) + } else if String::from_utf8_lossy(&output.stderr).contains("No such process") { + Ok(false) + } else { + Err(format!( + "SDK owned group {} {signal}: {}: {}{}", + self.child.id(), + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + )) + } + } +} + +impl Drop for TestSdkWorker { + fn drop(&mut self) { + if let Err(error) = self.terminate() { + eprintln!("SDK cleanup also failed: {error}"); + } + if let Err(error) = self.output() { + eprintln!("SDK stdout artifact {}: {error}", self.stdout.display()); + } + } +} diff --git a/e2e-test/tests/access_iceberg/gc/notifications/matrix.rs b/e2e-test/tests/access_iceberg/gc/notifications/matrix.rs index df69e69e1..0d152f5a9 100644 --- a/e2e-test/tests/access_iceberg/gc/notifications/matrix.rs +++ b/e2e-test/tests/access_iceberg/gc/notifications/matrix.rs @@ -33,20 +33,33 @@ struct TestMatrix { async fn restarted_cleaners_finish_large_and_sparse_backlogs_beside_an_invalid_catalog() { let stack = TestNativeStorage::new(); let (store, context, table) = seed_table(&stack).await; - let large = TestLargeRange::write(&stack, store.clone(), context, table).await; - let records = TestMatrixRecords::seed(store, context, table, large.file.clone()).await; + let started = Instant::now(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(720); + let pressure = TestGcPressure::prepare_before(deadline).await.unwrap(); + let large = tokio::time::timeout_at( + deadline, + TestLargeRange::write(&stack, store.clone(), context, table), + ) + .await + .expect("GC matrix large range seed exceeded the existing case deadline"); + let records = tokio::time::timeout_at( + deadline, + TestMatrixRecords::seed(store, context, table, large.file.clone()), + ) + .await + .expect("GC matrix record seed exceeded the existing case deadline"); let mut check = TestMatrix { records, large, servers: Vec::new(), - pressure: None, - started: Instant::now(), + pressure: Some(pressure), + started, observations: Vec::new(), last_revision: None, restarts: 0, }; - let result = tokio::time::timeout( - Duration::from_secs(720), + let result = tokio::time::timeout_at( + deadline, std::panic::AssertUnwindSafe(check.run(&stack.seeds)).catch_unwind(), ) .await; @@ -124,8 +137,12 @@ impl TestMatrix { let multipart = self.records.multipart().await?; let deleted = task.deleted; self.observe(&task, multipart.as_ref()); - if let Some(pressure) = &mut self.pressure { - pressure.observe(deleted).await?; + let pressure = self + .pressure + .as_mut() + .ok_or("prepared foreground pressure is missing")?; + if pressure.released() { + pressure.observe(deleted, task.phase == GcPhase::Complete).await?; } else if deleted > 0 { let remaining = self.records.remaining().await?; if task.phase == GcPhase::Complete || remaining == 0 { @@ -135,7 +152,7 @@ impl TestMatrix { "gc_matrix_foreground_start {}", serde_json::json!({"elapsed_ms":self.started.elapsed().as_millis(),"deleted":deleted,"remaining_canonical":remaining,"revision":task.revision,"phase":format!("{:?}",task.phase)}) ); - self.pressure = Some(TestGcPressure::start(&self.servers[0].origin, seeds.to_vec())?); + pressure.release(seeds.to_vec())?; } if task.phase == GcPhase::Complete && self.records.sparse_complete().await? diff --git a/e2e-test/tests/access_iceberg/gc/sdk_pressure.py b/e2e-test/tests/access_iceberg/gc/sdk_pressure.py index 9e90f4f9f..1020221c2 100644 --- a/e2e-test/tests/access_iceberg/gc/sdk_pressure.py +++ b/e2e-test/tests/access_iceberg/gc/sdk_pressure.py @@ -1,17 +1,40 @@ # Copyright 2026-present Gian # Licensed under the Apache License, Version 2.0. -import sys, json, time +import sys, json, time, traceback from concurrent.futures import ThreadPoolExecutor -import requests from pathlib import Path -from pyiceberg.catalog import load_catalog -from pyiceberg.schema import Schema -from pyiceberg.types import LongType, NestedField -control = Path(sys.argv[5]) if len(sys.argv) > 5 else None +control = None + +def wait_for_release(directory, worker): + (directory / f"ready-{worker}").touch() + while True: + failures = sorted(directory.glob("failed-*")) + if failures: + raise RuntimeError(f"foreground peer failed: {failures[0].read_text()}") + if (directory / "stop").exists(): + raise RuntimeError("foreground cancelled before release") + if (directory / "run").exists(): + return + time.sleep(0.01) + +def guarded_run(worker): + try: + return run(worker) + except BaseException: + if control and not any(control.glob("failed-*")): + temporary = control / f"failure-{worker}.tmp" + temporary.write_text(traceback.format_exc()) + temporary.replace(control / f"failed-{worker}") + raise def run(worker): + import requests + from pyiceberg.catalog import load_catalog + from pyiceberg.schema import Schema + from pyiceberg.types import LongType, NestedField + samples = {} windows = {} def measure(operation, call): @@ -27,14 +50,16 @@ def measure(operation, call): namespace = (f"{prefix}-{worker}",) measure("create_namespace", lambda: catalog.create_namespace(namespace)) if control: - (control / f"ready-{worker}").touch() - while not (control / "run").exists(): - if (control / "stop").exists(): - raise RuntimeError("foreground cancelled before release") - time.sleep(0.01) + wait_for_release(control, worker) count = int(sys.argv[2]) if len(sys.argv) > 2 else 3 + interval = float(sys.argv[3]) if len(sys.argv) > 3 else 0 + # Keep paced workers spread across a cycle so a short deletion interval + # cannot repeatedly fall into all four workers' inter-table gaps. + if interval: + time.sleep(worker * interval / 4) index = 0 - while index < count or (count == 0 and (index < 3 or not (control / "stop").exists())): + minimum = count if count else 3 + while index < minimum or (control and not (control / "stop").exists()): identifier = namespace + (f"events-{index}",) table = measure("create_table", lambda: catalog.create_table(identifier, Schema(NestedField(field_id=1, name="id", field_type=LongType(), required=True)))) measure("commit", lambda: table.transaction().set_properties({"gc-probe": str(index)}).commit_transaction()) @@ -58,20 +83,26 @@ def read_metadata(): temporary.write_text(str(index)) temporary.replace(control / f"progress-{worker}") if len(sys.argv) > 3: - time.sleep(float(sys.argv[3])) + time.sleep(interval) measure("drop_namespace", lambda: catalog.drop_namespace(namespace)) return samples, windows -with ThreadPoolExecutor(max_workers=4) as executor: - results = list(executor.map(run, range(4))) -combined = {} -combined_windows = {} -for result, windows in results: - for operation, samples in result.items(): - combined.setdefault(operation, []).extend(samples) - combined_windows.setdefault(operation, []).extend(windows.get(operation, [])) -report = {} -for operation, samples in combined.items(): - ordered = sorted(samples) - report[operation] = {"raw_ns":samples,"count":len(samples),"p50_ns":ordered[(len(ordered)-1)//2],"p99_ns":ordered[(99*len(ordered)+99)//100-1],"windows_unix_ns":combined_windows[operation]} -print("gc_foreground_official_sdk " + json.dumps(report), flush=True) +def main(): + global control + control = Path(sys.argv[5]) if len(sys.argv) > 5 else None + with ThreadPoolExecutor(max_workers=4) as executor: + results = list(executor.map(guarded_run, range(4))) + combined = {} + combined_windows = {} + for result, windows in results: + for operation, samples in result.items(): + combined.setdefault(operation, []).extend(samples) + combined_windows.setdefault(operation, []).extend(windows.get(operation, [])) + report = {} + for operation, samples in combined.items(): + ordered = sorted(samples) + report[operation] = {"raw_ns":samples,"count":len(samples),"p50_ns":ordered[(len(ordered)-1)//2],"p99_ns":ordered[(99*len(ordered)+99)//100-1],"windows_unix_ns":combined_windows[operation]} + print("gc_foreground_official_sdk " + json.dumps(report), flush=True) + +if __name__ == "__main__": + main() diff --git a/e2e-test/tests/access_iceberg/signed_file.rs b/e2e-test/tests/access_iceberg/signed_file.rs index 4840084b1..b009a65a5 100644 --- a/e2e-test/tests/access_iceberg/signed_file.rs +++ b/e2e-test/tests/access_iceberg/signed_file.rs @@ -53,6 +53,7 @@ fn canonical_query(query: &str) -> String { .join("&") } +#[derive(Clone)] pub struct TestFileClient { pub client: Client, pub credentials: crowdb_access_iceberg::file::FileCredentials, diff --git a/e2e-test/tests/admission_profile_test.py b/e2e-test/tests/admission_profile_test.py index 7967587f9..8eab5714f 100644 --- a/e2e-test/tests/admission_profile_test.py +++ b/e2e-test/tests/admission_profile_test.py @@ -9,6 +9,7 @@ from pathlib import Path import sys import tempfile +import tomllib import unittest from unittest.mock import patch import run @@ -92,3 +93,8 @@ def reply(*arguments): self.assertNotEqual(policies[0], policies[1]) self.assertTrue(all(policy.parent == Path(directory) for policy in policies)) self.assertEqual(policies[0].read_text(), policies[1].read_text()) + accepted = tomllib.loads(policies[0].read_text()) + enabled = [service for service in accepted["services"] if service["enabled"]] + self.assertEqual(accepted["storage"], {"mode": "diskless"}) + self.assertEqual(enabled, [{"kind": "paxos-kv", "enabled": True, + "listeners": {"http_port": 10000, "rpc_port": 10100}}]) diff --git a/e2e-test/tests/browser_batch_provenance_test.py b/e2e-test/tests/browser_batch_provenance_test.py new file mode 100644 index 000000000..32782d887 --- /dev/null +++ b/e2e-test/tests/browser_batch_provenance_test.py @@ -0,0 +1,140 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Batch setup failure does not impersonate a dispatched browser failure.""" + +import contextlib +import io +import json +import tempfile +import unittest +from pathlib import Path + +from browser_batch import run_browser_batches +from fixture import Cluster +from report import write_report + + +class TestCluster(Cluster): + def __init__(self, directory, case): + super().__init__('image', directory) + self.case_name = case + self.start_error = None + self.collection_error = None + self.cleanup_error = None + self.collections = self.cleanups = 0 + (self.artifacts / 'case.json').write_text(json.dumps({ + 'case': case, 'scope': 'console.native.browser.lifecycle'})) + + def start(self): + if self.start_error: + raise self.start_error + return self + + def diagnostics(self): + self.collections += 1 + if self.collection_error: + raise self.collection_error + + def close(self): + self.cleanups += 1 + if self.cleanup_error: + raise self.cleanup_error + self.closed = True + self.write_timings() + + +class BrowserBatchProvenanceTest(unittest.TestCase): + def run_failed_startup(self, directory, *, cleanup_error=None, collection_error=None): + peer = TestCluster(directory, 'prepared-peer') + failed = TestCluster(directory, 'failed-startup') + failed.start_error = RuntimeError('original startup rejected') + peer.cleanup_error, peer.collection_error = cleanup_error, collection_error + output = io.StringIO() + with contextlib.redirect_stdout(output): + errors = run_browser_batches([peer, failed], 2, lambda case: case, + lambda *_: self.fail('client was dispatched')) + self.assertIs(errors[0], failed.start_error) + self.assertEqual((peer.collections, peer.cleanups), (1, 1)) + self.assertEqual((failed.collections, failed.cleanups), (1, 1)) + self.assertTrue(failed.closed) + self.assertEqual(failed.timings['status'], 'failed') + self.assertEqual(output.getvalue().count('[FAILED CASE] failed-startup:'), 1) + self.assertIn('original startup rejected', output.getvalue()) + self.assertIn('[NOT EXECUTED CASE] prepared-peer:', output.getvalue()) + return peer, errors, output.getvalue() + + def test_prepared_peer_is_not_executed_and_cannot_be_reported_as_pass(self): + with tempfile.TemporaryDirectory() as directory: + peer, errors, output = self.run_failed_startup(directory) + self.assertEqual(len(errors), 1) + self.assertTrue(peer.closed) + self.assertNotIn('[FAILED CASE] prepared-peer:', output) + self.assertEqual(peer.timings['status'], 'not_executed') + self.assertIn('failed-startup', peer.timings['not_executed_reason']) + timing = json.loads((peer.artifacts / 'timing.json').read_text()) + self.assertEqual(timing['status'], 'not_executed') + write_report(Path(directory), {'status': 'failed'}) + report = json.loads((Path(directory) / 'results.json').read_text()) + self.assertTrue(all(case['status'] != 'pass' for case in report['cases'])) + self.assertEqual({case['case']: case['status'] for case in report['cases']}, { + 'prepared-peer': 'not_executed', 'failed-startup': 'failed'}) + self.assertEqual(report['timings']['status'], 'failed') + + def test_peer_cleanup_failure_retains_original_startup_and_its_own_error(self): + with tempfile.TemporaryDirectory() as directory: + cleanup = RuntimeError('peer cleanup rejected') + peer, errors, output = self.run_failed_startup(directory, cleanup_error=cleanup) + self.assertEqual(errors[1:], [cleanup]) + self.assertEqual(peer.timings['status'], 'failed') + self.assertIn('[FAILED CASE] prepared-peer: peer cleanup rejected', output) + self.assertNotIn('[FAILED CASE] prepared-peer: original startup rejected', output) + + def test_peer_collection_failure_still_cleans_and_retains_its_own_error(self): + with tempfile.TemporaryDirectory() as directory: + collection = OSError('peer artifact disk full') + peer, errors, output = self.run_failed_startup(directory, collection_error=collection) + self.assertEqual(errors[1:], [collection]) + self.assertTrue(peer.closed) + self.assertEqual(peer.timings['status'], 'failed') + self.assertIn('[FAILED CASE] prepared-peer: peer artifact disk full', output) + self.assertNotIn('[FAILED CASE] prepared-peer: original startup rejected', output) + + def test_peer_collection_and_cleanup_failures_are_both_retained(self): + with tempfile.TemporaryDirectory() as directory: + collection = OSError('peer artifact disk full') + cleanup = RuntimeError('peer cleanup rejected') + peer, errors, output = self.run_failed_startup( + directory, collection_error=collection, cleanup_error=cleanup) + self.assertEqual(errors[1:], [collection]) + self.assertIn(str(cleanup), collection.__notes__) + self.assertIs(collection.__cause__, cleanup) + self.assertEqual(peer.timings['status'], 'failed') + self.assertIn('[FAILED CASE] prepared-peer: peer artifact disk full', output) + self.assertIn('[FAILED CASE] prepared-peer: peer cleanup rejected', output) + + def test_real_client_and_cleanup_failures_remain_independent_of_successful_peer(self): + with tempfile.TemporaryDirectory() as directory: + failed, peer = TestCluster(directory, 'failed-client'), TestCluster(directory, 'passed-client') + failure = RuntimeError('browser assertion failed') + failed.cleanup_error = RuntimeError('owned browser cleanup failed') + executed = [] + + def execute(case, cluster): + executed.append(case) + if cluster is failed: + raise failure + + output = io.StringIO() + with contextlib.redirect_stdout(output): + errors = run_browser_batches([failed, peer], 2, lambda case: case, execute) + self.assertCountEqual(executed, [failed, peer]) + self.assertEqual(errors, [failure]) + self.assertEqual(failed.timings['status'], 'failed') + self.assertEqual(peer.timings['status'], 'pass') + self.assertIn('[FAILED CASE] failed-client: browser assertion failed', output.getvalue()) + self.assertEqual(output.getvalue().count('[FAILED CASE] failed-client:'), 1) + self.assertIn('[CLEANUP ERROR] failed-client: owned browser cleanup failed', output.getvalue()) + self.assertIn('owned browser cleanup failed', failure.__notes__) + self.assertNotIn('[FAILED CASE] passed-client:', output.getvalue()) + self.assertNotIn('[CLEANUP ERROR] passed-client:', output.getvalue()) + self.assertNotIn('[NOT EXECUTED CASE]', output.getvalue()) diff --git a/e2e-test/tests/browser_batch_test.py b/e2e-test/tests/browser_batch_test.py index 50334eed0..9d744f187 100644 --- a/e2e-test/tests/browser_batch_test.py +++ b/e2e-test/tests/browser_batch_test.py @@ -21,6 +21,8 @@ def __exit__(self, kind, error, traceback): owner.events.append(('close', case)) owner.failures[case] = error owner.active.remove(case) + if case == owner.cleanup_failure: + raise RuntimeError('cleanup rejected') return Fixture() @@ -28,6 +30,7 @@ def setUp(self): self.events = [] self.active = set() self.start_failure = None + self.cleanup_failure = None self.failures = {} def test_clients_share_stable_bounded_batch_and_teardown_precedes_next_batch(self): @@ -62,6 +65,17 @@ def test_partial_startup_closes_started_peer_without_dispatching_clients(self): self.assertEqual([str(error) for error in errors], ['startup rejected']) self.assertEqual(self.events, [('start', 0), ('start', 1), ('close', 0)]) self.assertEqual(self.active, set()) + self.assertEqual(str(self.failures[0]), + 'browser client not executed: batch startup failed in 1') + + def test_cleanup_failure_is_not_attributed_to_a_successful_peer(self): + self.cleanup_failure = 1 + errors = run_browser_batches([0, 1], 2, self.fixture, lambda *_: None) + self.assertEqual([str(error) for error in errors], ['cleanup rejected']) + self.assertEqual(self.events[-2:], [('close', 1), ('close', 0)]) + self.assertEqual(self.active, set()) + self.assertIsNone(self.failures[0]) + self.assertIsNone(self.failures[1]) def test_fail_fast_closes_current_batch_before_skipping_later_fixtures(self): def execute(case, fixture): @@ -73,6 +87,14 @@ def execute(case, fixture): self.assertEqual(self.events, [('start', 0), ('close', 0)]) self.assertEqual(self.active, set()) + def test_fail_fast_startup_closes_peer_before_skipping_later_batch(self): + self.start_failure = 1 + errors = run_browser_batches([0, 1, 2], 2, self.fixture, + lambda *_: self.fail('client was dispatched'), fail_fast=True) + self.assertEqual([str(error) for error in errors], ['startup rejected']) + self.assertEqual(self.events, [('start', 0), ('start', 1), ('close', 0)]) + self.assertEqual(self.active, set()) + def test_large_fixture_does_not_serialize_later_small_browser_batches(self): def execute(case, fixture): self.assertEqual(self.active, {0} if case == 0 else {1, 2}) diff --git a/e2e-test/tests/chunkdb/allocation.rs b/e2e-test/tests/chunkdb/allocation.rs index 7595c3e7b..6e2dab7e9 100644 --- a/e2e-test/tests/chunkdb/allocation.rs +++ b/e2e-test/tests/chunkdb/allocation.rs @@ -3,6 +3,9 @@ use crowdb_chunkdb::allocator::DiskdbClientPool; use crowdb_kv_client::ServiceRegistryClient; +use std::sync::atomic::Ordering; + +use crate::common::rpc_response_proxy::TestResponseProxy; #[tokio::test] #[ignore = "requires the owned Diskdb container fixture"] @@ -10,7 +13,6 @@ async fn allocation_connect_failure_reroutes_but_lost_reply_does_not() { use crate::common::hardware::{DG_ID, UNIT_SIZE_BYTES}; use crowdb_diskdb_client::{DiskdbClientError, DiskdbRpcTransport}; use crowdb_protocol::common::ChunkId; - use tokio::io::{AsyncReadExt, AsyncWriteExt}; let pool = DiskdbClientPool::new(ServiceRegistryClient::new(crate::common::hardware::kv_client())); pool.refresh_endpoints().await.expect("discover live owner"); @@ -34,26 +36,9 @@ async fn allocation_connect_failure_reroutes_but_lost_reply_does_not() { // The proxy forwards the allocation, then closes without its response. // Group-0 still advertises the direct endpoint, so an unsafe replay would // allocate another set there. - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let proxy = listener.local_addr().unwrap().to_string(); - let upstream_endpoint = endpoint.clone(); - let forward = tokio::spawn(async move { - let (downstream, _) = listener.accept().await.unwrap(); - let upstream = tokio::net::TcpStream::connect(upstream_endpoint).await.unwrap(); - let (mut input, mut output) = downstream.into_split(); - let (mut responses, mut requests) = upstream.into_split(); - let send = tokio::io::copy(&mut input, &mut requests); - let discard = async { - let mut buffer = [0; 8192]; - assert!(responses.read(&mut buffer).await.unwrap() > 0); - output.shutdown().await.unwrap(); - }; - tokio::select! { - _ = send => panic!("request stream ended before response"), - () = discard => {} - } - }); - pool.replace_endpoints_for_tests(vec![(DG_ID, proxy)]); + let proxy = TestResponseProxy::start(endpoint.clone()).await; + proxy.armed.store(true, Ordering::SeqCst); + pool.replace_endpoints_for_tests(vec![(DG_ID, proxy.endpoint.clone())]); let result = pool .allocate_blocks_reusing_disks(DG_ID, 1, 1, &ChunkId { high: 0, low: 42 }) .await; @@ -61,7 +46,8 @@ async fn allocation_connect_failure_reroutes_but_lost_reply_does_not() { matches!(result, Err(DiskdbClientError::Unreachable(_))), "{result:?}" ); - forward.await.unwrap(); + assert_eq!(proxy.forwarded_handshakes.load(Ordering::SeqCst), 1); + assert_eq!(proxy.dropped.load(Ordering::SeqCst), 1); let transport = DiskdbRpcTransport::new(); let info = transport diff --git a/e2e-test/tests/common.rs b/e2e-test/tests/common.rs index ff6461a39..e66cfc8bf 100644 --- a/e2e-test/tests/common.rs +++ b/e2e-test/tests/common.rs @@ -6,6 +6,7 @@ pub(crate) mod hardware; mod logging; pub(crate) mod management; pub(crate) mod native_s3; +pub(crate) mod rpc_response_proxy; mod test_client; use bytes::Bytes; diff --git a/e2e-test/tests/common/control.rs b/e2e-test/tests/common/control.rs index d4d0c21d4..fa86e9246 100644 --- a/e2e-test/tests/common/control.rs +++ b/e2e-test/tests/common/control.rs @@ -26,6 +26,14 @@ pub(crate) async fn native_browser() -> Value { request_with_timeout(json!({"action": "run-native-browser"}), Duration::from_secs(305)).await } +pub(crate) async fn native_browser_with_journal(bucket: &str) -> Value { + request_with_timeout( + json!({"action":"run-native-browser", "journal_bucket":bucket}), + Duration::from_secs(305), + ) + .await +} + pub(crate) async fn native_browser_with_chunks(chunks: Value) -> Value { request_with_timeout( json!({"action":"run-native-browser", "chunks": chunks}), diff --git a/e2e-test/tests/console/common/rpc_response_proxy.rs b/e2e-test/tests/common/rpc_response_proxy.rs similarity index 98% rename from e2e-test/tests/console/common/rpc_response_proxy.rs rename to e2e-test/tests/common/rpc_response_proxy.rs index 86eb0b7c1..d8743a606 100644 --- a/e2e-test/tests/console/common/rpc_response_proxy.rs +++ b/e2e-test/tests/common/rpc_response_proxy.rs @@ -125,7 +125,7 @@ async fn forward( return; } // Capability advertisements precede application replies and do not - // establish that the conditional write has completed. + // establish that the application request has completed. let handshake = message_type == 0xfffe; if !handshake && armed.swap(false, Ordering::SeqCst) { dropped.fetch_add(1, Ordering::SeqCst); diff --git a/e2e-test/tests/console.rs b/e2e-test/tests/console.rs index 8640cf481..f85c29bc6 100644 --- a/e2e-test/tests/console.rs +++ b/e2e-test/tests/console.rs @@ -7,8 +7,7 @@ mod group_publication; mod hardware; mod logical_publication; -#[path = "console/common/rpc_response_proxy.rs"] -mod rpc_response_proxy; +use crate::common::rpc_response_proxy; #[path = "console/common/bootstrap_authority.rs"] mod bootstrap_authority; diff --git a/e2e-test/tests/console/browser_dataset.rs b/e2e-test/tests/console/browser_dataset.rs index 6d1e7f532..6d5e68317 100644 --- a/e2e-test/tests/console/browser_dataset.rs +++ b/e2e-test/tests/console/browser_dataset.rs @@ -1,37 +1,45 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -async fn run() { +async fn run(browser_phase: &str) { + let phase = std::env::var("CROWDB_E2E_PHASE").expect("owned Dataset client phase"); + assert!( + phase == "prepare" || phase == browser_phase, + "unexpected Dataset phase: {phase}" + ); let context = super::native_context::TestNativeContext::connect().await; - super::native_storage::start(&context).await; - // The first split leaves the normal partition-count policy unsatisfied. - // Finish bootstrap placement before the bounded Dataset reads begin. - super::native_balance::settle(&context).await; + if phase == "prepare" { + super::native_storage::start(&context).await; + // Finish the full production placement policy in the preparation phase. + super::native_balance::settle(&context).await; + return; + } + super::native_balance::settle_existing(&context).await; let result = crate::common::control::native_browser().await; assert_eq!(result["passed"], true); } #[tokio::test] #[ignore = "requires native Dataset and installed browser"] async fn browse() { - run().await; + run("browse").await; } #[tokio::test] #[ignore = "requires native Dataset and installed browser"] async fn import() { - run().await; + run("import").await; } #[tokio::test] #[ignore = "requires native Dataset and installed browser"] async fn split_revisions() { - run().await; + run("split_revisions").await; } #[tokio::test] #[ignore = "requires native Dataset and installed browser"] async fn recovery() { - run().await; + run("recovery").await; } #[tokio::test] #[ignore = "requires native Dataset and installed browser"] async fn capabilities() { - run().await; + run("capabilities").await; } diff --git a/e2e-test/tests/console/deployment.rs b/e2e-test/tests/console/deployment.rs index 0b8d27a8a..b7d93976d 100644 --- a/e2e-test/tests/console/deployment.rs +++ b/e2e-test/tests/console/deployment.rs @@ -36,6 +36,7 @@ async fn concurrent_admissions_allocate_unique_stable_ids_through_group_zero() { .await .unwrap(); let node = |uuid: &str, host: &str| NodeRecord { + web_port: 9090, discovery_id: uuid.into(), node_id: 0, physical_host_id: "same-host".into(), @@ -173,6 +174,7 @@ async fn authenticated_node_update_preserves_ids_and_converges_from_another_cons .await .unwrap(); let source = NodeRecord { + web_port: 9090, discovery_id: uuid::Uuid::new_v4().to_string(), node_id: 7, physical_host_id: "same-host".into(), diff --git a/e2e-test/tests/console/native_balance.rs b/e2e-test/tests/console/native_balance.rs index dc701c7ad..6008510ac 100644 --- a/e2e-test/tests/console/native_balance.rs +++ b/e2e-test/tests/console/native_balance.rs @@ -20,6 +20,7 @@ use std::{ }; pub(super) async fn settle(context: &TestNativeContext) { + let started = Instant::now(); let kv = context.kv.clone(); let source = Arc::new(Group0ChunkKvRangeCatalogSource::from_shared(kv.clone())); let registry = ServiceRegistryClient::from_shared(kv.clone()); @@ -29,9 +30,56 @@ pub(super) async fn settle(context: &TestNativeContext) { Arc::new(ChunkKvRpcTransport::new(64, 1, 2)), ) .unwrap(); + let phase = Instant::now(); + eprintln!("[PHASE] native production balance seed started: 512 records"); seed_values(&client).await; - wait_balanced(&source, ®istry, &kv, Instant::now()).await; + eprintln!( + "[PHASE] native production balance seed completed: phase_ms={}, total_ms={}", + phase.elapsed().as_millis(), + started.elapsed().as_millis() + ); + let phase = Instant::now(); + let catalog = wait_balanced(&source, ®istry, &kv, phase).await; + eprintln!( + "[PHASE] native production balance converged: phase_ms={}, total_ms={}, partitions={}, owners={}", + phase.elapsed().as_millis(), + started.elapsed().as_millis(), + catalog.entries().len(), + counts(&catalog).len() + ); + let phase = Instant::now(); verify_values(&client).await; + eprintln!( + "[PHASE] native production balance verified 512 records: phase_ms={}, total_ms={}", + phase.elapsed().as_millis(), + started.elapsed().as_millis() + ); +} + +pub(super) async fn settle_existing(context: &TestNativeContext) { + let source = Group0ChunkKvRangeCatalogSource::from_shared(context.kv.clone()); + let started = Instant::now(); + let mut interval = tokio::time::interval(Duration::from_millis(100)); + loop { + interval.tick().await; + let catalog = load(&source).await; + if catalog.entries().iter().all(|entry| { + entry.state == ChunkKvRangeCatalogPartitionState::Serving + && entry.transition_id.is_none() + && entry.artifact.tail_overlay.is_none() + }) && placement_ready(&context.kv, catalog.generation()).await + { + eprintln!( + "[PHASE] native fixture placement settled in {:?}", + started.elapsed() + ); + return; + } + assert!( + started.elapsed() < Duration::from_secs(40), + "native fixture's normal placement did not settle" + ); + } } async fn seed_values(client: &ChunkKvClient) { diff --git a/e2e-test/tests/console/native_browser.rs b/e2e-test/tests/console/native_browser.rs index 60a10bd5a..80b2a4c40 100644 --- a/e2e-test/tests/console/native_browser.rs +++ b/e2e-test/tests/console/native_browser.rs @@ -44,7 +44,33 @@ async fn storage_windows_and_access_recovery() { #[tokio::test] #[ignore = "requires normal storage and a host browser for owner recovery"] async fn journal_identity_and_owner_recovery() { - browse_storage().await; + let context = TestNativeContext::connect().await; + super::native_storage::start(&context).await; + let bucket = format!("journal-{}", uuid::Uuid::new_v4().simple()); + for (path, body) in [ + (format!("/api/access/s3/{bucket}"), ""), + ( + format!("/api/access/s3/{bucket}/value"), + "native journal observation", + ), + ] { + let response = context + .http + .put(format!("{}{path}", context.base)) + .body(body) + .timeout(std::time::Duration::from_secs(10)) + .send() + .await + .unwrap(); + assert!( + response.status().is_success(), + "journal fixture write: {}", + response.text().await.unwrap() + ); + } + super::native_balance::settle_existing(&context).await; + let result = crate::common::control::native_browser_with_journal(&bucket).await; + assert_eq!(result["passed"], true); } #[tokio::test] @@ -109,7 +135,12 @@ async fn browse_diskdb() { #[tokio::test] #[ignore = "requires normal storage and owned container PID observations"] async fn auxiliary_process_menus() { - browse_storage().await; + let context = TestNativeContext::connect().await; + super::native_storage::start(&context).await; + // Finish initial placement before testing one service outage at a time. + super::native_balance::settle_existing(&context).await; + let result = crate::common::control::native_browser().await; + assert_eq!(result["passed"], true); } #[tokio::test] diff --git a/e2e-test/tests/console_cli_inventory_test.py b/e2e-test/tests/console_cli_inventory_test.py new file mode 100644 index 000000000..d3e69af4b --- /dev/null +++ b/e2e-test/tests/console_cli_inventory_test.py @@ -0,0 +1,124 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. + +import copy +import json +import unittest +from unittest.mock import Mock, patch + +from console_cli_target import CliAttachedCluster +from fixture import LABEL + + +class AttachedInventoryTest(unittest.TestCase): + def setUp(self): + self.cluster = object.__new__(CliAttachedCluster) + self.cluster.servers = ["owned-1", "owned-2", "owned-3"] + self.cluster.addresses = ["192.0.2.1", "192.0.2.2", "192.0.2.3"] + self.cluster.run_id = "fixture" + self.cluster.cluster_id = "cluster" + self.cluster.image = "image" + self.exited = set() + self.containers = {name: {"Id": name, "Image": "image", "State": {"Running": True}, + "Config": {"Labels": {LABEL: "fixture"}}} for name in self.cluster.servers} + self.status, self.services = {}, [] + kinds = ("paxos-kv", "diskdb", "diskio", "chunkdb", "chunk-kv", "access-server") + for node, name in enumerate(self.cluster.servers, 1): + processes = {"web": {"pid": 20, "generation": 1, "operation_id": None}} + for number, kind in enumerate(kinds, 1): + identity = f"{kind}-{node}" + processes["kv" if kind == "paxos-kv" else identity] = { + "pid": 100 + number, "generation": 1, "operation_id": f"operation-{node}-{number}"} + self.services.append({"id": identity, "node_id": node, "service_type": kind, + "endpoint": f"192.0.2.{node}:{10000 + number}", "health": "unknown", "start_pending": False}) + self.status[name] = {"services": processes} + self.cluster.command = Mock(side_effect=self.command) + self.request = patch("console_cli_target.request", side_effect=lambda *_: copy.deepcopy(self.services)).start() + self.addCleanup(patch.stopall) + + def command(self, action, name, *arguments): + if action == "inspect": + return json.dumps([self.containers[name]]) + if action == "exec" and arguments[0] == "sh": + self.assertEqual(arguments[3], "owned-service-pids") + if any((name, int(pid)) in self.exited for pid in arguments[4:]): + raise RuntimeError("owned service PID is not live") + return "" + self.assertEqual((action, arguments[0]), ("exec", "cat")) + if arguments[1] == "/opt/crowdb/data/node-binding.json": + return json.dumps({"node_id": self.cluster.servers.index(name) + 1, + "bootstrap": {"cluster_id": "cluster"}}) + self.assertEqual(arguments[1], "/opt/crowdb/run/status/monitor.json") + return json.dumps(self.status[name]) + + def test_cold_cache_does_not_hide_process_or_registration_changes(self): + before = self.cluster.inventory() + self.assertEqual(len(before["processes"]), 21) + self.services[0].update(pid=101, pid_node_id=1, health="up") + self.assertEqual(self.cluster.inventory(), before) + self.status["owned-1"]["services"]["chunk-kv-1"]["pid"] += 1 + self.assertNotEqual(self.cluster.inventory(), before) + self.status["owned-1"]["services"]["chunk-kv-1"]["pid"] -= 1 + self.services[0]["endpoint"] = "192.0.2.1:29999" + self.assertNotEqual(self.cluster.inventory(), before) + + def test_foreign_container_is_rejected_before_monitor_access(self): + self.containers["owned-1"]["Config"]["Labels"][LABEL] = "foreign" + with self.assertRaisesRegex(AssertionError, "ownership"): + self.cluster.inventory() + self.assertFalse(any(call.args[0] == "exec" for call in self.cluster.command.call_args_list)) + + def test_invalid_real_pid_and_wrong_cached_namespace_are_rejected(self): + self.status["owned-1"]["services"]["web"]["pid"] = None + with self.assertRaisesRegex(AssertionError, "PID"): + self.cluster.inventory() + self.status["owned-1"]["services"]["web"]["pid"] = 20 + self.services[0].update(pid=101, pid_node_id=2) + with self.assertRaisesRegex(AssertionError, "namespace"): + self.cluster.inventory() + + def test_old_positive_status_does_not_hide_process_exit_or_missing_web(self): + self.exited.add(("owned-1", 20)) + with self.assertRaisesRegex(RuntimeError, "not live"): + self.cluster.inventory() + self.exited.clear() + del self.status["owned-1"]["services"]["web"] + with self.assertRaisesRegex(AssertionError, "service is absent"): + self.cluster.inventory() + + +class AttachedReadinessTest(unittest.TestCase): + def setUp(self): + self.cluster = object.__new__(CliAttachedCluster) + self.cluster.cluster_id = "cluster" + self.cluster.ready_deadline = 280 + self.cluster.invoke = Mock(return_value=json.dumps({ + "phase": "ready", "observation": {"cluster_id": "cluster"}})) + + def test_final_check_uses_only_the_remaining_startup_budget(self): + with patch("console_cli_target.time.monotonic", return_value=245.5): + self.cluster.check_ready(endpoint="http://node3:9090") + self.cluster.invoke.assert_called_once_with( + "cluster", "check", "--wait", "34", "--json", endpoint="http://node3:9090") + + def test_expired_budget_does_not_dispatch_another_check(self): + with patch("console_cli_target.time.monotonic", return_value=280), \ + self.assertRaisesRegex(AssertionError, "original readiness deadline"): + self.cluster.check_ready() + self.cluster.invoke.assert_not_called() + + def test_check_failure_is_not_retried_and_wrong_cluster_is_rejected(self): + self.cluster.invoke.side_effect = AssertionError("degraded") + with patch("console_cli_target.time.monotonic", return_value=250): + with self.assertRaisesRegex(AssertionError, "degraded"): + self.cluster.check_ready() + self.cluster.invoke.assert_called_once() + self.cluster.invoke.side_effect = None + self.cluster.invoke.return_value = json.dumps({ + "phase": "ready", "observation": {"cluster_id": "foreign"}}) + with self.assertRaises(AssertionError): + self.cluster.check_ready() + + +if __name__ == "__main__": + unittest.main() diff --git a/e2e-test/tests/console_cli_readiness_test.py b/e2e-test/tests/console_cli_readiness_test.py new file mode 100644 index 000000000..63100af76 --- /dev/null +++ b/e2e-test/tests/console_cli_readiness_test.py @@ -0,0 +1,109 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. + +import copy +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from console_cli_readiness import check_progress, check_unavailable, observe_unavailable + + +def failed(generation=24, checked=10000, age=2705, state="current"): + return {"ready": False, "live": True, + "chunks": "unavailable" if state == "current" else "unknown", + "details": {"chunks": "DiskIO read exceeded the readiness deadline"}, + "reasons": ["chunks"] if state == "current" else ["observation-expired"], + "observation": {"generation": generation, "checked_at_unix_ms": checked, + "age_ms": age, "max_age_ms": 3000, "state": state}} + + +def advancing(): + return [{"responses": [failed(24, 10000), failed(26, 10500), failed(25, 10100, 3071, "expired")]}, + {"responses": [failed(25, 12000), failed(27, 12500), failed(26, 12100)]}, + {"responses": [failed(26, 14000), failed(28, 14500), failed(27, 14100)]}] + + +class UnavailableProgressTest(unittest.TestCase): + def test_expired_negative_evidence_keeps_truthful_age_and_continuous_progress(self): + check_progress(advancing(), 3) + + def test_three_second_quantized_boundary_allows_both_states(self): + for state in ("current", "expired"): + check_unavailable([failed(age=3000, state=state)], 1) + + def test_stalled_sampler_fails_even_when_its_last_sample_is_current(self): + history = [{"responses": [failed()]} for _ in range(4)] + with self.assertRaisesRegex(AssertionError, "Sampler did not advance twice"): + check_progress(history, 1) + + def test_one_stalled_endpoint_cannot_hide_behind_healthy_siblings(self): + history = advancing() + for sample in history: + sample["responses"][1] = failed() + with self.assertRaisesRegex(AssertionError, "Sampler did not advance twice"): + check_progress(history, 3) + + def test_missing_error_ready_or_inconsistent_freshness_samples_are_rejected(self): + for value in ({"error": "health request timed out"}, + dict(failed(), ready=True), + failed(age=3071), + failed(age=2999, state="expired"), + dict(failed(), details={})): + with self.subTest(value=value), self.assertRaises(AssertionError): + check_unavailable([value], 1) + + def test_timestamp_refresh_without_a_new_generation_and_regression_are_rejected(self): + for generation, checked in ((24, 11000), (23, 12000), (25, 9999)): + history = copy.deepcopy(advancing()) + history[1]["responses"][0] = failed(generation, checked) + with self.subTest(generation=generation, checked=checked), self.assertRaises(AssertionError): + check_progress(history, 3) + + def test_missing_endpoint_is_rejected(self): + history = advancing() + history[1]["responses"].pop() + with self.assertRaises(AssertionError): + check_progress(history, 3) + + def test_observation_uses_the_last_eight_seconds_of_the_original_outage(self): + clock = [0.0] + calls = [] + + def sleep(seconds): + clock[0] += seconds + + def snapshot(): + calls.append(clock[0]) + generation = int(clock[0] // 2) + 1 + clock[0] += 0.1 + return [failed(generation, generation * 2000) for _ in range(3)] + + with tempfile.TemporaryDirectory() as directory, \ + patch("console_cli_readiness.time.monotonic", side_effect=lambda: clock[0]), \ + patch("console_cli_readiness.time.sleep", side_effect=sleep): + observe_unavailable(directory, snapshot, 3) + report = json.loads((Path(directory) / "access-health-paused-history.json").read_text()) + self.assertEqual(calls[0], 32) + self.assertLessEqual(calls[-1], 40) + self.assertAlmostEqual(clock[0], 40.1) + self.assertIsNone(report["failure"]) + self.assertEqual(len(report["samples"]), len(calls)) + + def test_failed_sample_is_retained_and_not_retried_until_it_passes(self): + with tempfile.TemporaryDirectory() as directory, \ + patch("console_cli_readiness.time.sleep"), \ + self.assertRaises(AssertionError): + try: + observe_unavailable(directory, lambda: [{"error": "unreachable"}], 1) + finally: + report = json.loads((Path(directory) / "access-health-paused-history.json").read_text()) + self.assertEqual(len(report["samples"]), 1) + self.assertEqual(report["samples"][0]["responses"], [{"error": "unreachable"}]) + self.assertIsNotNone(report["failure"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/e2e-test/tests/console_cli_test.py b/e2e-test/tests/console_cli_test.py index 32ec73096..71b16868c 100644 --- a/e2e-test/tests/console_cli_test.py +++ b/e2e-test/tests/console_cli_test.py @@ -9,6 +9,8 @@ from console_cli_fixture import CliContainerCluster, LABEL, OWNER_LABEL from console_cli_target import CliAttachedCluster +from console_cli_recovery import health_snapshot +from console_cli_io_fault import DiskIoOutage from fixture import Cluster @@ -25,7 +27,7 @@ def test_diagnostics_export_bounded_access_metrics_before_teardown(self): opener.open.return_value = MagicMock() opener.open.return_value.__enter__.return_value = response cluster.diagnostics() - opener.open.assert_called_once_with("http://192.0.2.1:9091/_crowdb/metrics", timeout=2) + opener.open.assert_called_once_with("http://192.0.2.1:16000/_crowdb/metrics", timeout=2) response.read.assert_called_once_with(1024 * 1024 + 1) collect.assert_called_once() self.assertIn('phase="diskio"', (cluster.artifacts / "access-metrics-0.prom").read_text()) @@ -97,3 +99,71 @@ def test_recovery_observation_rejects_invalid_node_before_inspection(self): with self.assertRaisesRegex(ValueError, "outside"): cluster.read_owned("node-binding.json", index) cluster.command.assert_not_called() + + +class CliAcceptedListenersTest(unittest.TestCase): + def test_health_uses_each_owned_nodes_accepted_listener(self): + for port in (9094, 19000, 19002): + with self.subTest(port=port), tempfile.TemporaryDirectory() as directory: + cluster = object.__new__(CliContainerCluster) + cluster.servers = ["owned"] + cluster.addresses = ["192.0.2.1"] + cluster.artifacts = Path(directory) + cluster.inspect_owned = Mock(return_value={"Id": "verified"}) + cluster.read_owned = Mock(return_value={"setup": {"services": [ + {"kind": "access-server", "enabled": True, "listeners": { + "health_port": port, "s3_port": 16000}}]}}) + cluster.command = Mock(return_value='{"ready":true,"observation":{"state":"current"}}') + self.assertTrue(health_snapshot(cluster, "test")[0]["ready"]) + cluster.read_owned.assert_called_once_with("node-policy.json", 0) + cluster.command.assert_called_once_with("exec", "verified", "curl", "--silent", + "--show-error", "--max-time", "3", f"http://192.0.2.1:{port}/_crowdb/health/ready") + + +class CliFaultCleanupTest(unittest.TestCase): + def outage(self): + outage = object.__new__(DiskIoOutage) + outage.chain = "OWNED_FAULT" + outage.nodes = [("a", 13000), ("b", 13000), ("c", 13000)] + outage._prepare_tools = Mock() + return outage + + def test_rule_removal_failure_still_flushes_all_owned_chains(self): + outage = self.outage() + calls = [] + + def command(node, *arguments): + calls.append((node, arguments)) + if node[0] == "c" and arguments[0] == "-D": + raise RuntimeError("one removal failed") + + outage._command = command + with self.assertRaisesRegex(RuntimeError, "one removal failed"): + with outage: + pass + for operation in ("-F", "-X"): + self.assertEqual({node[0] for node, args in calls if args[0] == operation}, {"a", "b", "c"}) + + def test_partial_setup_failure_cleans_previous_and_partial_nodes(self): + outage = self.outage() + calls = [] + + def command(node, *arguments): + calls.append((node, arguments)) + if node[0] == "b" and arguments[0] == "-A": + raise RuntimeError("rule setup failed") + + outage._command = command + with self.assertRaisesRegex(RuntimeError, "rule setup failed"): + with outage: + self.fail("partial setup must not admit the workload") + self.assertEqual({node[0] for node, args in calls if args[0] == "-X"}, {"a", "b"}) + self.assertFalse(any(node[0] == "c" for node, _args in calls)) + + def test_network_fault_rejects_host_network_before_executing_helpers(self): + outage = self.outage() + outage.cluster = Mock(run_id="owned") + outage.cluster.inspect_owned.return_value = {"HostConfig": {"NetworkMode": "host"}} + with self.assertRaisesRegex(ValueError, "owned CLI bridge"): + DiskIoOutage._command(outage, outage.nodes[0], "-N", outage.chain) + outage.cluster.command.assert_not_called() diff --git a/e2e-test/tests/dataset_observation_test.py b/e2e-test/tests/dataset_observation_test.py index 8815e86bf..232c40999 100644 --- a/e2e-test/tests/dataset_observation_test.py +++ b/e2e-test/tests/dataset_observation_test.py @@ -132,7 +132,7 @@ def test_six_boundaries_preserve_original_counts_and_explicit_nonadditive_scope( def test_fragmented_unicode_and_ordinary_output_are_continuously_preserved(self): rows = list(messages()) - rows[0]["observation"]["unavailable"]["extra"] = "雪" + rows[0]["observation"]["unavailable"]["extra"] = "\u96ea" raw = b"ordinary report\n" + b"".join(encoded(row) for row in rows) + b"ordinary EOF without newline" child = TestChild(chunks=[raw[index:index + 17] for index in range(0, len(raw), 17)]) with tempfile.TemporaryDirectory() as directory: diff --git a/e2e-test/tests/deployment.rs b/e2e-test/tests/deployment.rs index 6a7793c36..6183c9053 100644 --- a/e2e-test/tests/deployment.rs +++ b/e2e-test/tests/deployment.rs @@ -18,6 +18,9 @@ mod validation; #[tokio::test] #[ignore = "requires the normal release-monitor container fixture"] async fn selected_kv_services_recover_acknowledged_data() { + if std::env::var("CROWDB_E2E_PHASE").as_deref() == Ok("write") { + kv_bootstrap::create_witness_group().await; + } witness(false).await; } diff --git a/e2e-test/tests/deployment/kv_bootstrap.rs b/e2e-test/tests/deployment/kv_bootstrap.rs index d601feebd..295064038 100644 --- a/e2e-test/tests/deployment/kv_bootstrap.rs +++ b/e2e-test/tests/deployment/kv_bootstrap.rs @@ -33,6 +33,10 @@ async fn monitor_creates_real_kv_groups_then_validates_after_restart() { ) .await; assert_eq!(result["bootstrap"], accepted["bootstrap"]); + if write { + create_witness_group().await; + } + wait_for_witness_group().await; for group in [0, 1] { let ready = request( &http, @@ -70,3 +74,76 @@ async fn monitor_creates_real_kv_groups_then_validates_after_restart() { assert_eq!(retained["bootstrap"], accepted["bootstrap"]); super::witness_with_write(false, write).await; } + +pub(super) async fn create_witness_group() { + use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, ReadMode}; + + let origin = format!("http://{}:10000", crate::common::management::addresses(1)[0]); + let kv = CrowdbKvClient::new(ClientConfig::new(vec![origin.clone()])); + assert!(matches!( + kv.get( + 0, + 0, + crowdb_console_shared::deployment::preparation::DEFAULT_PREPARATION_KEY, + ReadMode::Linearizable, + None, + ) + .await + .unwrap(), + GetOutcome::NotFound + )); + let http = client(); + // KV-only admission creates Group 0. This case explicitly requests its + // ordinary witness group instead of depending on storage preparation. + request( + &http, + &origin, + Method::GET, + "/stores/0/groups/1/ready", + Value::Null, + 404, + ) + .await; + request( + &http, + &origin, + Method::POST, + "/stores/0/groups", + json!({"group_id":1,"replica_id":1}), + 201, + ) + .await; + wait_for_witness_group().await; +} + +async fn wait_for_witness_group() { + use std::time::{Duration, Instant}; + + let origin = format!("http://{}:10000", crate::common::management::addresses(1)[0]); + let http = client(); + let deadline = Instant::now() + Duration::from_secs(4); + loop { + let response = http + .get(format!("{origin}/stores/0/groups/1/ready")) + .send() + .await + .expect("witness group readiness request"); + let status = response.status(); + let ready: Value = response.json().await.expect("witness group readiness JSON"); + assert!( + matches!(status.as_u16(), 200 | 503), + "unexpected witness readiness HTTP {status}: {ready}" + ); + assert_eq!(status.as_u16() == 200, ready["ready"] == true, "{ready}"); + if ready["ready"] == true { + assert_eq!(ready["leader_id"], 1); + assert_eq!(ready["voting_replicas"], 1); + return; + } + assert!( + Instant::now() < deadline, + "explicit witness group did not become ready: {ready}" + ); + tokio::time::sleep(Duration::from_millis(25)).await; + } +} diff --git a/e2e-test/tests/fixture_test.py b/e2e-test/tests/fixture_test.py index 3f3ddf739..5eea1294e 100644 --- a/e2e-test/tests/fixture_test.py +++ b/e2e-test/tests/fixture_test.py @@ -2,6 +2,8 @@ # Licensed under the Apache License, Version 2.0. """Failure ownership and resource contracts without a container daemon.""" +import contextlib +import io import json import tempfile import subprocess @@ -107,9 +109,35 @@ def test_cleanup_failure_preserves_the_original_error(self): with tempfile.TemporaryDirectory() as directory: daemon = Daemon() daemon.fail_cleanup = True - with self.assertRaisesRegex(RuntimeError, "first server failed") as caught, Cluster("image", directory, daemon): + cluster = Cluster("image", directory, daemon) + cluster.case_name = "owned-startup" + output = io.StringIO() + with contextlib.redirect_stdout(output), \ + self.assertRaisesRegex(RuntimeError, "first server failed") as caught, cluster: pass self.assertIn("owned cleanup failed", " ".join(caught.exception.__notes__)) + self.assertEqual(output.getvalue().count("[FAILED CASE] owned-startup:"), 1) + self.assertIn("[CLEANUP ERROR] owned-startup: owned cleanup failed", output.getvalue()) + self.assertIn("daemon cleanup unavailable", output.getvalue()) + + def test_cleanup_failure_after_client_success_prints_the_actual_case_and_error(self): + with tempfile.TemporaryDirectory() as directory: + daemon = Daemon() + daemon.fail_cleanup = True + cluster = Cluster("image", directory, daemon) + cluster.case_name = "owned-successful-client" + cluster.network_created = True + daemon.resources[("network", cluster.run_id)] = cluster.run_id + output = io.StringIO() + with patch.object(cluster, "start", return_value=cluster), \ + contextlib.redirect_stdout(output), \ + self.assertRaisesRegex(RuntimeError, "owned cleanup failed"), cluster: + pass + self.assertIn("[FAILED CASE] owned-successful-client: owned cleanup failed", output.getvalue()) + self.assertIn("daemon cleanup unavailable", output.getvalue()) + self.assertIn(str(cluster.artifacts), output.getvalue()) + self.assertEqual(cluster.timings["status"], "failed") + self.assertEqual(json.loads((cluster.artifacts / "timing.json").read_text())["status"], "failed") def test_interruption_tears_down_and_keeps_other_fixture(self): with tempfile.TemporaryDirectory() as directory: diff --git a/e2e-test/tests/gc_sdk_pressure_test.py b/e2e-test/tests/gc_sdk_pressure_test.py new file mode 100644 index 000000000..de41ea50b --- /dev/null +++ b/e2e-test/tests/gc_sdk_pressure_test.py @@ -0,0 +1,121 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Control-protocol regressions only; no SDK, endpoint, process or container.""" + +import ast +from concurrent.futures import ThreadPoolExecutor +import os +from pathlib import Path +import tempfile +import threading +import types +import unittest +from unittest.mock import patch + + +def pressure_module(): + source = Path(os.environ.get("CROWDB_GC_PRESSURE_TEST_SOURCE", Path(__file__).parent / + "access_iceberg/gc/sdk_pressure.py")) + tree = ast.parse(source.read_text(), filename=str(source)) + # Load the real control functions without importing or executing an SDK. + declarations = [node for node in tree.body if isinstance(node, ast.FunctionDef) + or isinstance(node, (ast.Import, ast.ImportFrom)) and not ( + isinstance(node, ast.ImportFrom) and node.module.startswith("pyiceberg") + or isinstance(node, ast.Import) and any(alias.name == "requests" for alias in node.names))] + module = types.ModuleType("gc_sdk_pressure") + exec(compile(ast.Module(body=declarations, type_ignores=[]), str(source), "exec"), module.__dict__) + module.control = None + return module + + +class GcSdkPressureControlTest(unittest.TestCase): + def setUp(self): + self.pressure = pressure_module() + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) + + def test_all_four_workers_park_without_doing_post_ready_work_until_release(self): + parked = threading.Barrier(5) + release_check = threading.Event() + completed = [] + + def observed_sleep(_seconds): + parked.wait(timeout=2) + if not release_check.wait(timeout=2): + raise TimeoutError("test did not release parked control workers") + + def worker(index): + self.pressure.wait_for_release(self.root, index) + completed.append(index) + + with ThreadPoolExecutor(max_workers=4) as executor: + futures = [] + try: + with patch.object(self.pressure.time, "sleep", side_effect=observed_sleep): + futures = [executor.submit(worker, index) for index in range(4)] + parked.wait(timeout=2) + self.assertEqual(sorted(path.name for path in self.root.glob("ready-*")), + ["ready-0", "ready-1", "ready-2", "ready-3"]) + self.assertFalse((self.root / "run").exists()) + self.assertEqual(completed, []) + (self.root / "run").touch() + release_check.set() + for future in futures: + future.result(timeout=2) + finally: + (self.root / "stop").touch() + release_check.set() + self.assertEqual(sorted(completed), [0, 1, 2, 3]) + + def test_stop_prevents_release_even_if_run_marker_also_exists(self): + (self.root / "stop").touch() + (self.root / "run").touch() + with self.assertRaisesRegex(RuntimeError, "cancelled before release"): + self.pressure.wait_for_release(self.root, 1) + + def test_original_worker_error_is_published_and_unblocks_parked_peer(self): + self.pressure.control = self.root + with patch.object(self.pressure, "run", side_effect=ValueError("original catalog error")): + with self.assertRaisesRegex(ValueError, "original catalog error"): + self.pressure.guarded_run(2) + failure = (self.root / "failed-2").read_text() + self.assertIn("ValueError: original catalog error", failure) + self.assertIn("Traceback", failure) + with self.assertRaisesRegex(RuntimeError, "foreground peer failed:.*"): + self.pressure.wait_for_release(self.root, 0) + + def test_failure_prevents_release_even_when_run_marker_arrives(self): + (self.root / "failed-3").write_text("original SDK HTTP failure") + (self.root / "run").touch() + with self.assertRaisesRegex(RuntimeError, "original SDK HTTP failure"): + self.pressure.wait_for_release(self.root, 2) + + def test_failure_is_not_visible_until_the_complete_traceback_is_published(self): + self.pressure.control = self.root + real_write = Path.write_text + + def interrupted_write(path, text): + real_write(path, text[:len(text) // 2]) + self.assertEqual(list(self.root.glob("failed-*")), []) + return real_write(path, text) + + with patch.object(Path, "write_text", new=interrupted_write), \ + patch.object(self.pressure, "run", side_effect=ValueError("whole original error")): + with self.assertRaisesRegex(ValueError, "whole original error"): + self.pressure.guarded_run(1) + self.assertTrue((self.root / "failed-1").read_text().endswith("ValueError: whole original error\n")) + + def test_peer_cancellation_keeps_the_original_failure_record(self): + self.pressure.control = self.root + original = "ValueError: actual SDK request failed" + (self.root / "failed-3").write_text(original) + with patch.object(self.pressure, "run", side_effect=RuntimeError("foreground peer failed")): + with self.assertRaisesRegex(RuntimeError, "foreground peer failed"): + self.pressure.guarded_run(0) + self.assertEqual(sorted(path.name for path in self.root.glob("failed-*")), ["failed-3"]) + self.assertEqual((self.root / "failed-3").read_text(), original) + + +if __name__ == "__main__": + unittest.main() diff --git a/e2e-test/tests/journal_request_stability_test.py b/e2e-test/tests/journal_request_stability_test.py new file mode 100644 index 000000000..81420e5ac --- /dev/null +++ b/e2e-test/tests/journal_request_stability_test.py @@ -0,0 +1,20 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Exercise exact journal request/response ownership without intercepting E2E APIs.""" + +from pathlib import Path +import subprocess +import unittest + + +class JournalRequestStabilityTest(unittest.TestCase): + def test_journal_runtime_request_identity_and_refresh_order(self): + script = Path(__file__).with_name("journal_runtime_request_test.mjs") + result = subprocess.run(["pixi", "run", "node", "--experimental-strip-types", "--test", str(script)], + capture_output=True, text=True, timeout=60) + print(result.stdout + result.stderr, end="", flush=True) + self.assertEqual(result.returncode, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/e2e-test/tests/journal_runtime_request_test.mjs b/e2e-test/tests/journal_runtime_request_test.mjs new file mode 100644 index 000000000..c6d623953 --- /dev/null +++ b/e2e-test/tests/journal_runtime_request_test.mjs @@ -0,0 +1,74 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import { test } from 'node:test'; +import { journalRuntimeResponse, matchesJournalRuntimeRequest } from '../clients/console-ui/e2e/fixtures/journalRuntimeRequest.ts'; + +const target = { origin: 'http://owned', id: 'a'.repeat(32), epoch: '9007199254740993', generation: '9007199254740997', page: 2, offset: 17 }; +const url = () => new URL('/api/chunk-kv/runtime?' + new URLSearchParams({ id: target.id, epoch: target.epoch, + generation: target.generation, page: String(target.page), offset: String(target.offset), stream_offset: '0' }), target.origin); +const request = (address = url(), method = 'GET') => ({ url: () => String(address), method: () => method, + failure: () => ({ errorText: 'owned request aborted' }) }); +const response = (owner, finished = async () => null) => ({ request: () => owner, finished }); +const remaining = () => 1_000; + +test('journal matcher rejects another partition and every old resource or cursor component', () => { + assert.equal(matchesJournalRuntimeRequest(request(), target), true); + for (const [name, value] of [['id', 'b'.repeat(32)], ['epoch', '1'], ['generation', '2'], + ['page', '0'], ['offset', '0'], ['stream_offset', '20'], ['stream_generation', 'old'], + ['page_path', ''], ['entry_offset', '0']]) { + const changed = url(); changed.searchParams.set(name, value); + assert.equal(matchesJournalRuntimeRequest(request(changed), target), false, name); + } + const duplicate = url(); duplicate.searchParams.append('id', target.id); + assert.equal(matchesJournalRuntimeRequest(request(duplicate), target), false); + const foreign = url(); foreign.hostname = 'foreign'; + assert.equal(matchesJournalRuntimeRequest(request(foreign), target), false); + assert.equal(matchesJournalRuntimeRequest(request(url(), 'POST'), target), false); +}); + +test('an automatic response started before Refresh cannot satisfy its request observation', async () => { + const page = new EventEmitter(); + const automatic = request(); + page.emit('request', automatic); + const refreshed = request(); + const wrong = url(); wrong.searchParams.set('id', 'b'.repeat(32)); + const unrelated = request(wrong); + const observed = await journalRuntimeResponse(page, target, async () => { + page.emit('response', response(automatic)); + page.emit('request', unrelated); page.emit('response', response(unrelated)); + page.emit('request', refreshed); page.emit('response', response(refreshed)); + }, remaining); + assert.equal(observed.request(), refreshed); + assert.equal(page.listenerCount('request'), 0); + assert.equal(page.listenerCount('response'), 0); + assert.equal(page.listenerCount('requestfailed'), 0); +}); + +test('Refresh ignores an old cursor and waits for its exact response body to finish', async () => { + const page = new EventEmitter(); + const oldUrl = url(); oldUrl.searchParams.set('stream_generation', 'old'); + const old = request(oldUrl), refreshed = request(); + let finish; + const body = new Promise(resolve => { finish = resolve; }); + let completed = false; + const pending = journalRuntimeResponse(page, target, async () => { + page.emit('request', old); page.emit('response', response(old)); + page.emit('request', refreshed); page.emit('response', response(refreshed, () => body)); + }, remaining).then(value => { completed = true; return value; }); + await new Promise(resolve => setImmediate(resolve)); + assert.equal(completed, false); + finish(null); + assert.equal((await pending).request(), refreshed); +}); + +test('request failure or action failure removes all listeners and keeps the real error', async () => { + const page = new EventEmitter(), failed = request(); + await assert.rejects(journalRuntimeResponse(page, target, async () => { + page.emit('request', failed); page.emit('requestfailed', failed); + }, remaining), /owned request aborted/); + await assert.rejects(journalRuntimeResponse(page, target, async () => { throw new Error('selection click failed'); }, remaining), /selection click failed/); + for (const name of ['request', 'response', 'requestfailed']) assert.equal(page.listenerCount(name), 0); +}); diff --git a/e2e-test/tests/kv/reconfiguration.rs b/e2e-test/tests/kv/reconfiguration.rs index 34e3b07b2..847aeceae 100644 --- a/e2e-test/tests/kv/reconfiguration.rs +++ b/e2e-test/tests/kv/reconfiguration.rs @@ -3,7 +3,7 @@ use super::cluster::{leader, topology}; use crate::common::management::{addresses, client, origin, request}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, KvRpcTransport, ReadMode}; use reqwest::{Client, Method}; use serde_json::{json, Value}; use std::time::{Duration, Instant}; @@ -32,37 +32,145 @@ pub(super) async fn wire_initial(http: &Client, ips: &[String]) { } } -async fn expand(http: &Client, ips: &[String]) { - for (index, ip) in ips.iter().enumerate() { - let peers = if index < 3 { 3..5 } else { 0..5 }; - for peer in peers.filter(|peer| *peer != index) { - request( +async fn membership(http: &Client, ip: &str, method: Method, path: &str, body: Value, epoch: u64) { + let response = http + .request(method, format!("{}{path}", origin(ip))) + .header("x-crowdb-membership-epoch", epoch.to_string()) + .json(&body) + .send() + .await + .unwrap(); + let status = response.status(); + let text = response.text().await.unwrap(); + assert_eq!(status.as_u16(), 200, "{path}: {text}"); +} + +async fn applied(http: &Client, ip: &str) -> u64 { + topology(http, ip).await["stores"][0]["groups"][0]["read_state"]["contiguous_applied"] + .as_u64() + .expect("group applied frontier") +} + +async fn imported_value(ip: &str, required: u64) { + let response = KvRpcTransport::new() + .send_get( + &format!("http://{ip}:7001"), + b"k1", + 9201, + 9201, + 1, + crowdb_kv::rpc::ReadMode::MinSlot, + required, + ) + .await + .unwrap(); + assert!(response.ok, "snapshot local read failed: {}", response.error); + assert!(!response.not_found, "snapshot omitted acknowledged k1"); + assert_eq!(response.value.as_ref(), b"v1"); +} + +async fn add_member(http: &Client, ips: &[String], index: usize, epoch: u64) { + let source = leader(http, &ips[..index], 1).await; + let required = applied(http, &ips[source]).await; + request( + http, + &origin(&ips[index]), + Method::POST, + &format!("/stores/{index}/groups/1/join"), + json!({"replica_id": index + 1, "peer_endpoint": format!("{}:7001", ips[source])}), + 201, + ) + .await; + // Snapshot state must exist before this replica participates in any quorum. + assert!(applied(http, &ips[index]).await >= required); + imported_value(&ips[index], required).await; + for (peer, ip) in ips[..index].iter().enumerate() { + request( + http, + &origin(ip), + Method::POST, + &format!("/stores/{peer}/groups/1/remotes"), + json!([{"replica_id": index + 1, "endpoint": format!("{}:7001", ips[index]), "voting": false}]), + 200, + ) + .await; + } + let remotes: Vec<_> = ips[..index] + .iter() + .enumerate() + .map(|(peer, ip)| json!({"replica_id": peer + 1, "endpoint": format!("{ip}:7001"), "voting": true})) + .collect(); + membership( + http, + &ips[index], + Method::POST, + &format!("/stores/{index}/groups/1/remotes"), + json!(remotes), + epoch, + ) + .await; + let required = applied(http, &ips[source]).await; + let deadline = Instant::now() + Duration::from_secs(10); + while applied(http, &ips[index]).await < required { + assert!( + Instant::now() < deadline, + "joining member did not catch up before promotion" + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } + for (peer, ip) in ips[..index].iter().enumerate() { + membership( + http, + ip, + Method::POST, + &format!("/stores/{peer}/groups/1/remotes"), + json!([{"replica_id": index + 1, "endpoint": format!("{}:7001", ips[index]), "voting": true}]), + epoch, + ) + .await; + } + leader(http, &ips[..=index], 1).await; +} + +async fn shrink(http: &Client, ips: &[String], mut epoch: u64) { + for removed in 0..2 { + let current = leader(http, &ips[removed..], 1).await + removed; + if current == removed { + let stepped = request( http, - &origin(ip), + &origin(&ips[removed]), Method::POST, - &format!("/stores/{index}/groups/1/remotes"), - json!([{"replica_id": peer + 1, "endpoint": format!("{}:7001", ips[peer])}]), + &format!("/stores/{removed}/groups/1/step-down?sync=true"), + json!({"reason": "decommission"}), 200, ) .await; + assert_eq!(stepped["accepted"], true); } - } -} - -async fn shrink(http: &Client, ips: &[String]) { - for (index, ip) in ips.iter().enumerate() { - let removed = if index < 2 { 2..5 } else { 0..2 }; - for peer in removed { - request( + epoch += 1; + // Retire while its old epoch is still current. The remaining members + // retain the old quorum until the single-member removal reaches them. + membership( + http, + &ips[removed], + Method::DELETE, + &format!("/stores/{removed}/groups/1"), + Value::Null, + epoch, + ) + .await; + for (index, ip) in ips.iter().enumerate().skip(removed + 1) { + membership( http, - &origin(ip), + ip, Method::DELETE, - &format!("/stores/{index}/groups/1/remotes/{}", peer + 1), + &format!("/stores/{index}/groups/1/remotes/{}", removed + 1), Value::Null, - 200, + epoch, ) .await; } + leader(http, &ips[removed + 1..], 1).await; } } @@ -90,15 +198,24 @@ async fn quorum_expands_and_shrinks_without_losing_writes() { let initial = kv(&ips[..3]); initial.put(0, 1, b"k1", b"v1", None).await.unwrap(); exact(&initial, 0, b"k1", b"v1").await; - expand(&http, &ips).await; + eprintln!("reconfiguration: acknowledged k1 before expansion"); + // First wiring bootstraps the original members at epoch zero. + let mut epoch = 0; + for index in 3..5 { + epoch += 1; + add_member(&http, &ips, index, epoch).await; + } leader(&http, &ips, 1).await; let expanded = kv(&ips); exact(&expanded, 0, b"k1", b"v1").await; expanded.put(0, 1, b"k2", b"v2", None).await.unwrap(); - shrink(&http, &ips).await; + eprintln!("reconfiguration: expansion preserves k1; acknowledged k2 before shrink"); + shrink(&http, &ips, epoch).await; leader(&http, &ips[2..], 1).await; let surviving = kv(&ips[2..]); + exact(&surviving, 2, b"k1", b"v1").await; exact(&surviving, 2, b"k2", b"v2").await; + eprintln!("reconfiguration: shrink preserves k1 and k2"); surviving.delete(2, 1, b"k1", None).await.unwrap(); assert!(matches!( surviving diff --git a/e2e-test/tests/kv/slot_publication.rs b/e2e-test/tests/kv/slot_publication.rs index 2705fb81e..33d970de3 100644 --- a/e2e-test/tests/kv/slot_publication.rs +++ b/e2e-test/tests/kv/slot_publication.rs @@ -121,8 +121,16 @@ async fn complete_epoch_publication_is_atomic_idempotent_and_storage_independent client.publish_service_epochs(&transfer), client.publish_service_epochs(&transfer) ); - assert!(a.is_ok() && b.is_ok(), "first: {a:?}; second: {b:?}"); + assert!( + matches!( + (&a, &b), + (Ok(()), Ok(()) | Err(crowdb_kv_client::Error::CasBusy)) + | (Err(crowdb_kv_client::Error::CasBusy), Ok(())) + ), + "first: {a:?}; second: {b:?}" + ); let after = client.read_service_snapshot().await.unwrap(); + assert_eq!(after.authority().head(), transfer.head()); for current in ChunkSlot::all() { assert_eq!(after.authority().owner(current), transfer.owner(current)); assert_eq!( @@ -134,6 +142,13 @@ async fn complete_epoch_publication_is_atomic_idempotent_and_storage_independent client.read_storage().await.unwrap().bindings(), storage.bindings() ); + // A busy caller reconciles the completed publication before submitting again. + client.publish_service_epochs(&transfer).await.unwrap(); + let repeated = client.read_service_snapshot().await.unwrap(); + assert_eq!(repeated.authority().head(), after.authority().head()); + assert_eq!(repeated.authority().bindings(), after.authority().bindings()); + assert_eq!(repeated.service().head(), after.service().head()); + assert_eq!(repeated.service().bindings(), after.service().bindings()); let reused = epochs(&service, 4, Some((slot, 1, 2))); assert!(client.publish_service_epochs(&reused).await.is_err()); let rollback = epochs(&service, 4, None); diff --git a/e2e-test/tests/oci_reference_test.py b/e2e-test/tests/oci_reference_test.py index c567a314b..270fda706 100644 --- a/e2e-test/tests/oci_reference_test.py +++ b/e2e-test/tests/oci_reference_test.py @@ -126,6 +126,9 @@ def test_recreated_names_reuse_exact_retained_volumes_on_the_accepted_new_networ self.assertIn('type=volume,source=owned-node1-data,target=/opt/crowdb/data', create) self.assertEqual(create[create.index('--network') + 1], 'owned-recovered') self.assertNotIn('owned-node1-cold-data', create) + published = [create[index+1] for index, arg in enumerate(create) if arg == '--publish'] + self.assertEqual(published, ['127.0.0.1:20000:9090', '127.0.0.1:20001:16000', + '127.0.0.1:20002:17000', '127.0.0.1:20003:18000']) def test_cold_network_scope_still_rejects_foreign_names_and_attachments(self): cluster = self.cluster() diff --git a/e2e-test/tests/oci_shortcuts_test.py b/e2e-test/tests/oci_shortcuts_test.py index 21811bb66..7929cf0ee 100644 --- a/e2e-test/tests/oci_shortcuts_test.py +++ b/e2e-test/tests/oci_shortcuts_test.py @@ -2,6 +2,9 @@ # Licensed under the Apache License, Version 2.0. from types import SimpleNamespace +from pathlib import Path +import subprocess +import tempfile import unittest from unittest.mock import patch @@ -9,6 +12,65 @@ class OciShortcutRetentionTest(unittest.TestCase): + def test_cleanup_only_selects_owned_receipts_after_success_or_failure(self): + for fail in (False, True): + with self.subTest(fail=fail), tempfile.TemporaryDirectory() as directory: + artifacts = Path(directory) + receipts = artifacts / 'shortcut-runtime/persistent/oci-clusters' + foreign = receipts / 'unrelated.json' + running, cleaned = {}, [] + + def invoke(command, **kwargs): + receipts.mkdir(parents=True, exist_ok=True) + if not foreign.exists(): + foreign.write_text('unrelated retained data') + task, arguments = command[2], command[3:] + output, code = '', 0 + if task == 'sh': + output = kwargs['env']['PATH'].split(':')[0] + '/crowdb-cli\n' + else: + self.assertNotIn('--all', arguments, 'fixture must never clean the shared daemon') + name = arguments[arguments.index('--cluster') + 1] + if task == 'oci-run': + code = 1 + elif task == 'oci-run-3c': + (receipts / (name + '.json')).write_text('owned') + running[name] = ['first', 'second', 'third'] + elif task == 'oci-clean': + cleaned.append((name, '--keep-data' in arguments)) + self.assertEqual((receipts / (name + '.json')).read_text(), 'owned') + running.pop(name, None) + if '--keep-data' not in arguments: + (receipts / (name + '.json')).unlink() + else: + self.fail(task) + return subprocess.CompletedProcess(command, code, output, '') + + def named(_cluster, _invoke, _ready, _ids, _credentials, one, other, _port): + if fail: + for name in (one, other): + (receipts / (name + '.json')).write_text('owned') + raise RuntimeError('injected preparation failure') + return 'one-id', 'other-id', [] + + cluster = SimpleNamespace(artifacts=artifacts, run_id='fixture-123456789012', + image='owned-image', image_id='owned-image-id', port_base=20000, + command=lambda *args: '\n'.join(running.get(args[-1].rsplit('=', 1)[1], []))) + ready = dict(phase='ready', observation=dict(policy=dict(mode='local'), + initial_voters=3, cluster_id='three-id')) + with patch.object(oci_shortcuts.subprocess, 'run', side_effect=invoke), \ + patch.object(oci_shortcuts, 'named_clusters', side_effect=named), \ + patch.object(oci_shortcuts, 'status', return_value=ready): + if fail: + with self.assertRaisesRegex(RuntimeError, 'injected preparation failure'): + oci_shortcuts.verify(cluster) + else: + oci_shortcuts.verify(cluster) + self.assertEqual(foreign.read_text(), 'unrelated retained data') + expected = [('pixi-123456789012-' + suffix, fail) + for suffix in (('a', 'b') if fail else ('c',))] + self.assertEqual(cleaned, expected) + def test_named_cluster_recreation_preserves_identity_ports_credentials_and_writes(self): saved, running, generations, racks = {}, {}, {}, {} diff --git a/e2e-test/tests/report_test.py b/e2e-test/tests/report_test.py index dfe0f3043..1ac5bc941 100644 --- a/e2e-test/tests/report_test.py +++ b/e2e-test/tests/report_test.py @@ -2,10 +2,16 @@ # Licensed under the Apache License, Version 2.0. """A passing client cannot hide an incomplete fixture lifecycle.""" +import contextlib +import importlib.util +import io import json +import subprocess +import sys import tempfile import unittest from pathlib import Path +from unittest.mock import patch from report import write_report @@ -14,6 +20,7 @@ class ReportTest(unittest.TestCase): def test_expected_nonzero_exit_is_passed_only_when_fixture_lifecycle_also_passed(self): for fixture_status, actual, expected, outcome in ( ("pass", 1, 1, "pass"), ("pass", 2, 1, "failed"), ("failed", 1, 1, "failed"), + ("not_executed", 0, 0, "not_executed"), ): with self.subTest(fixture=fixture_status, actual=actual, expected=expected): with tempfile.TemporaryDirectory() as directory: @@ -47,3 +54,36 @@ def test_failed_cleanup_is_a_failed_case_even_after_client_passed(self): self.assertEqual(results["cases"][0]["status"], "failed") self.assertEqual(results["timings"]["status"], "failed") self.assertEqual(results["slowest_clients"][0]["seconds"], 1.0) + + def test_measurement_counts_not_executed_separately_and_retains_batch_failure(self): + path = Path(__file__).resolve().parents[2] / 'tools/test-metrics/measure.py' + spec = importlib.util.spec_from_file_location('fixture_report_measure', path) + measure = importlib.util.module_from_spec(spec) + spec.loader.exec_module(measure) + + def fake_command(command, **kwargs): + if command[:3] == ['pixi', 'run', 'clean']: + return subprocess.CompletedProcess(command, 0) + output = Path(command[command.index('--artifacts') + 1]) / 'run-test' + output.mkdir(parents=True) + (output / 'results.json').write_text(json.dumps({ + 'timings': {'status': 'failed'}, + 'cases': [{'status': status} for status in ('pass', 'failed', 'not_executed')]})) + kwargs['stdout'].write('original startup rejected\n') + return subprocess.CompletedProcess(command, 1) + + with tempfile.TemporaryDirectory() as directory, \ + patch.object(measure, 'ROOT', Path(directory)), \ + patch.object(measure, 'TaskGraph') as graph, \ + patch.object(measure.subprocess, 'run', side_effect=fake_command), \ + patch.object(sys, 'argv', ['measure.py', 'test-e2e-console']), \ + contextlib.redirect_stdout(io.StringIO()) as console: + graph.return_value.tasks = {('default', 'test-e2e-console'): 'unused'} + self.assertEqual(measure.main(), 1) + record = json.loads((Path(directory) / + '.crowdb-runtime/artifacts/measure-tests/latest.json').read_text())[0] + self.assertEqual(record['passed'], 1) + self.assertEqual(record['failed'], 1) + self.assertEqual(record['not_executed'], 1) + self.assertEqual(record['exit_code'], 1) + self.assertIn('original startup rejected', console.getvalue()) diff --git a/e2e-test/tests/runner_progression_test.py b/e2e-test/tests/runner_progression_test.py index 78183eb6a..032b1230e 100644 --- a/e2e-test/tests/runner_progression_test.py +++ b/e2e-test/tests/runner_progression_test.py @@ -7,10 +7,55 @@ from pathlib import Path from unittest.mock import patch -from scenarios import Case, run_cases +from scenarios import Case, run_cases, select_cases class RunnerProgressionTest(unittest.TestCase): + def run_dataset_phases(self, fail_prepare=False): + events = [] + cases = select_cases(layer='console', scope='console.native.browser.dataset') + with tempfile.TemporaryDirectory() as temporary: + class Fixture: + def __init__(self, case): + self.name = case.name + self.artifacts = Path(temporary) / self.name + self.artifacts.mkdir() + + def __enter__(self): + return self + + def __exit__(self, *_): + events.append((self.name, 'close', self.client_timeout)) + + def client(self, _bundle, phase, name): + self_owner.assertEqual(name, self.name) + events.append((name, phase, self.client_timeout)) + if fail_prepare and phase == 'prepare': + raise ValueError('Dataset preparation failed') + + self_owner = self + with patch('scenarios.select_cases', return_value=cases), \ + patch('scenarios.allocate_cluster', side_effect=lambda case, *_: Fixture(case)): + if fail_prepare: + with self.assertRaisesRegex(ValueError, 'Dataset preparation failed'): + run_cases('image', temporary, 'bundle', 1, fail_fast=True) + else: + run_cases('image', temporary, 'bundle', 1) + return cases, events + + def test_dataset_preparation_and_browser_keep_separate_original_budgets(self): + cases, events = self.run_dataset_phases() + self.assertEqual(len(cases), 5) + self.assertEqual(events, [ + (case.name, phase, 360) + for case in cases + for phase in ('prepare', case.name.rsplit('::', 1)[-1], 'close') + ]) + + def test_dataset_preparation_failure_skips_browser_and_closes_fixture(self): + cases, events = self.run_dataset_phases(fail_prepare=True) + self.assertEqual(events, [(cases[0].name, phase, 360) for phase in ('prepare', 'close')]) + def test_ready_cli_executes_before_a_peer_finishes_starting(self): executed = threading.Event() events = [] diff --git a/e2e-test/tests/test_dispatch_test.py b/e2e-test/tests/test_dispatch_test.py index 8aa9999ad..217c28e5d 100644 --- a/e2e-test/tests/test_dispatch_test.py +++ b/e2e-test/tests/test_dispatch_test.py @@ -52,7 +52,9 @@ def test_compilation_cache_excludes_rootful_oci_state_and_runtime_artifacts(self self.assertIn('key: ${{ inputs.key }}', cache) self.assertIn('cmd-format: "pixi run {0}"', cache) self.assertIn('cache-workspace-crates: true', cache) - directories = cache.split('cache-directories: |', 1)[1].split('cmd-format:', 1)[0].split() + self.assertIn('cache-directories: ${{ inputs.directories }}', cache) + defaults = cache.split(' directories:\n', 1)[1].split('runs:', 1)[0] + directories = defaults.split(' default: |\n', 1)[1].split() self.assertEqual(set(directories), { 'target/debug', 'target/release', 'sdk/target/release', 'sdk/target/sdist/release', 'target/iceberg-sdk/release', 'app/crowdb-diskio/build', 'lib/crowdb-tree/build', 'lib/crowdb-rpc/build'}) @@ -65,6 +67,23 @@ def test_compilation_cache_excludes_rootful_oci_state_and_runtime_artifacts(self 'ContainerConsole', 'ContainerStorage', 'ContainerDeployment'): self.assertNotIn('uses: ./.github/actions/compilation-cache', self.job(job), job) + def test_access_cache_and_test_profile_bound_disk_without_narrowing_tasks(self): + cache = (ROOT / '.github/actions/compilation-cache/action.yml').read_text() + self.assertIn('default: release-ci-v1', cache) + self.assertIn("shared-key: ${{ inputs.family }}-${{ hashFiles('pixi.lock') }}", cache) + access = self.job('AccessTests') + self.assertRegex(access, r'(?m)^ CARGO_PROFILE_TEST_STRIP: debuginfo$') + self.assertRegex(access, r'(?m)^ CARGO_BUILD_JOBS: "4"$') + self.assertNotRegex(self.workflow, r'(?m)^ CARGO_PROFILE_TEST_STRIP:') + self.assertIn('family: access-ci-v2', access) + directories = re.search(r' directories: \|\n((?: [^\n]+\n)+)', access) + self.assertIsNotNone(directories) + self.assertEqual(directories[1].split(), [ + 'target/debug', 'sdk/target/release', 'sdk/target/sdist/release']) + self.assertEqual(re.findall(r'^ pixi run ([\w-]+)', access, re.M), [ + 'test-access', 'test-access-sdk', 'check-dataset-sdk-boundary', + 'sdk-fmt-check', 'sdk-lint', 'test-dataset-sdk', 'test-dataset-sdk-python']) + def test_only_binary_consumers_wait_for_shared_builds(self): for name in ('Lint', 'UITests', 'CppTests', 'UnitTests', 'StorageTests', 'AccessTests', 'ConsoleTests', 'ContainerImage', 'BuildClients'): diff --git a/lib/crowdb-access-iceberg/src/file/reader.rs b/lib/crowdb-access-iceberg/src/file/reader.rs index 8aa53a507..af98d0075 100644 --- a/lib/crowdb-access-iceberg/src/file/reader.rs +++ b/lib/crowdb-access-iceberg/src/file/reader.rs @@ -19,12 +19,19 @@ pub struct FileReader { end: u64, frame_bytes: usize, cached: Option<(u64, Vec)>, - leaf_directory: Option<(ChunkRoot, Vec)>, + leaf_directory: Option, digest: Option, locations: Option>, failed: bool, } +struct CachedDirectory { + root: ChunkRoot, + start: u64, + length: u64, + bytes: Vec, +} + struct ReadSource { owner: FileIdentity, length: u64, @@ -67,7 +74,12 @@ impl FileReader { }; verify_block(root, &bytes)?; ChunkDirectory::decode(&bytes, self.record.owner, root.height, self.record.length)?; - self.leaf_directory = Some((root.clone(), bytes)); + self.leaf_directory = Some(CachedDirectory { + root: root.clone(), + start: 0, + length: self.record.length, + bytes, + }); Ok(self) } /// # Errors @@ -158,7 +170,7 @@ impl FileReader { pub fn retained_directory_bytes(&self) -> usize { self.leaf_directory .as_ref() - .map_or(0, |(_, bytes)| bytes.capacity()) + .map_or(0, |directory| directory.bytes.capacity()) } /// # Errors @@ -214,11 +226,18 @@ impl FileReader { let FileContent::Chunks { root: Some(root) } = &self.record.content else { return Err(ValidationError::Record.into()); }; - let mut root = root.clone(); - let mut start = 0; - let mut length = self.record.length; + let (mut root, mut start, mut length) = self + .leaf_directory + .as_ref() + .filter(|directory| { + self.cursor >= directory.start && self.cursor - directory.start < directory.length + }) + .map_or_else( + || (root.clone(), 0, self.record.length), + |directory| (directory.root.clone(), directory.start, directory.length), + ); while root.height > 0 { - let directory = self.directory(&root, length).await?; + let directory = self.directory(&root, start, length).await?; let mut selected = None; for entry in directory.entries { if self.cursor - start < entry.length { @@ -239,11 +258,16 @@ impl FileReader { Ok((start, bytes)) } - async fn directory(&mut self, root: &ChunkRoot, length: u64) -> Result { - if let Some((selected, bytes)) = &self.leaf_directory { - if selected == root { + async fn directory( + &mut self, + root: &ChunkRoot, + start: u64, + length: u64, + ) -> Result { + if let Some(selected) = &self.leaf_directory { + if selected.root == *root && selected.start == start && selected.length == length { return Ok(ChunkDirectory::decode( - bytes, + &selected.bytes, self.record.owner, root.height, length, @@ -257,7 +281,12 @@ impl FileReader { verify_block(root, &bytes)?; let directory = ChunkDirectory::decode(&bytes, self.record.owner, root.height, length)?; if root.height == 1 { - self.leaf_directory = Some((root.clone(), bytes)); + self.leaf_directory = Some(CachedDirectory { + root: root.clone(), + start, + length, + bytes, + }); } Ok(directory) } diff --git a/lib/crowdb-access-iceberg/tests/file_json_test.rs b/lib/crowdb-access-iceberg/tests/file_json_test.rs index 6e1cbbbf8..50f8ae351 100644 --- a/lib/crowdb-access-iceberg/tests/file_json_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_json_test.rs @@ -41,7 +41,7 @@ async fn json_sealing_streams_large_strings_and_split_utf8_without_changing_cano let validator = JsonSealer::new(store.clone(), 2, 1024 * 1024, 64).unwrap(); for bytes in [ br#"{"n":1e+2,"a":[null,true,false],"s":"{\"x\"}"}"#.to_vec(), - "{\"字段\":\"冰😀\"}".as_bytes().to_vec(), + "{\"\u{5b57}\u{6bb5}\":\"\u{51b0}😀\"}".as_bytes().to_vec(), format!("{{\"large\":\"{}\"}}", "x".repeat(100_000)).into_bytes(), ] { let size = if bytes.len() < 100 { 7 } else { 2048 }; diff --git a/lib/crowdb-access-iceberg/tests/file_listing_test.rs b/lib/crowdb-access-iceberg/tests/file_listing_test.rs index 3e587c034..a2e716442 100644 --- a/lib/crowdb-access-iceberg/tests/file_listing_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_listing_test.rs @@ -124,7 +124,13 @@ async fn two_table_principals_cannot_discover_each_others_published_names() { async fn pages_and_delimiters_preserve_order_without_duplicate_groups() { let fixture = TestFile::new(common::TestStore::default()).await; let repository = FileRepository::new(fixture.store.clone()); - for relative in ["a/one.json", "a/two.json", "b/one.json", "c.json", "雪&<>.json"] { + for relative in [ + "a/one.json", + "a/two.json", + "b/one.json", + "c.json", + "\u{96ea}&<>.json", + ] { repository .publish(fixture.context, &fixture.record(relative, b"{}")) .await diff --git a/lib/crowdb-access-iceberg/tests/file_location_test.rs b/lib/crowdb-access-iceberg/tests/file_location_test.rs index a0a702588..034578c75 100644 --- a/lib/crowdb-access-iceberg/tests/file_location_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_location_test.rs @@ -16,7 +16,7 @@ fn canonical_locations_round_trip_without_names_or_key_normalization() { assert_eq!(table.to_string().parse::().unwrap(), table); for key in [ "metadata/v1.json", - "数据/a+b", + "\u{6570}\u{636e}/a+b", "a%2Fb", "a/b", "a//b", @@ -72,6 +72,6 @@ fn object_key_limit_counts_utf8_bytes_including_table_prefix() { assert_eq!(file.object_key().len(), MAX_OBJECT_KEY_BYTES); assert_eq!(file.to_string().parse::().unwrap(), file); assert!(table.file(&(key.clone() + "x")).is_err()); - assert!(table.file(&("x".repeat(available - 1) + "冰")).is_err()); + assert!(table.file(&("x".repeat(available - 1) + "\u{51b0}")).is_err()); assert!(FileLocation::from_object_key(&table.bucket(), &(file.object_key() + "x")).is_err()); } diff --git a/lib/crowdb-access-iceberg/tests/file_stream_test.rs b/lib/crowdb-access-iceberg/tests/file_stream_test.rs index 8db08a7c3..578fc8212 100644 --- a/lib/crowdb-access-iceberg/tests/file_stream_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_stream_test.rs @@ -111,8 +111,8 @@ async fn streaming_reuses_only_one_verified_leaf_directory_without_prefetch() { let input = vec![42; length]; writer.push(&input).await.unwrap(); let tree = writer.finish().await.unwrap(); - let parent_reads = if length > 256 { length } else { 0 }; let directory_reads = length.div_ceil(256); + let parent_reads = if length > 256 { directory_reads } else { 0 }; let mut reader = FileReader::new(store.clone(), record(owner, tree), None, 1).unwrap(); for expected in &input { assert_eq!(reader.next().await.unwrap().unwrap(), [*expected]); @@ -127,6 +127,30 @@ async fn streaming_reuses_only_one_verified_leaf_directory_without_prefetch() { } } +#[tokio::test] +async fn ranged_stream_reuses_the_current_directory_and_checks_the_next_boundary() { + let store = Arc::new(TestBlocks::default()); + let owner = owner(); + let input: Vec<_> = (0..513).map(|index| u8::try_from(index % 251).unwrap()).collect(); + let mut writer = FileTreeWriter::new(store.clone(), owner, 1).unwrap(); + writer.push(&input).await.unwrap(); + let mut reader = FileReader::new( + store.clone(), + record(owner, writer.finish().await.unwrap()), + Some(ByteRange { start: 254, end: 258 }), + 1, + ) + .unwrap(); + for expected in &input[254..256] { + assert_eq!(reader.next().await.unwrap().unwrap(), [*expected]); + } + assert_eq!(store.reads.load(Ordering::SeqCst), 4); + store.corrupt_reads.store(true, Ordering::SeqCst); + assert!(reader.next().await.is_err()); + assert!(reader.next().await.is_err()); + assert_eq!(store.reads.load(Ordering::SeqCst), 5); +} + #[tokio::test] async fn retained_directory_does_not_hide_new_leaf_corruption() { let store = Arc::new(TestBlocks::default()); diff --git a/lib/crowdb-access-iceberg/tests/key_test.rs b/lib/crowdb-access-iceberg/tests/key_test.rs index 21c07fa36..a72dfd88e 100644 --- a/lib/crowdb-access-iceberg/tests/key_test.rs +++ b/lib/crowdb-access-iceberg/tests/key_test.rs @@ -116,7 +116,7 @@ fn key_byte_limit_is_enforced_before_encoding() { #[test] fn name_fields_preserve_utf8_delimiters_and_parent_identity() { let parent = NamespaceId::random(); - for name in ["a/b", "a%1Fb", "数据.表", "a:1"] { + for name in ["a/b", "a%1Fb", "\u{6570}\u{636e}.\u{8868}", "a:1"] { let suffix = NameSuffix { parent: Some(parent), name, diff --git a/lib/crowdb-access-iceberg/tests/manifest_variant_test.rs b/lib/crowdb-access-iceberg/tests/manifest_variant_test.rs index 4f9844e72..6101c97d7 100644 --- a/lib/crowdb-access-iceberg/tests/manifest_variant_test.rs +++ b/lib/crowdb-access-iceberg/tests/manifest_variant_test.rs @@ -234,7 +234,7 @@ fn variant_paths_require_normalized_jsonpath_with_bounded_depth() { "$", "$['']", "$['user.name']", - "$['位置']['纬度']", + "$['\u{4f4d}\u{7f6e}']['\u{7eac}\u{5ea6}']", "$['a'][0]", "$[42]", r"$['a\'b']", diff --git a/lib/crowdb-access-iceberg/tests/namespace_create_test.rs b/lib/crowdb-access-iceberg/tests/namespace_create_test.rs index b666de46d..e0d5ee131 100644 --- a/lib/crowdb-access-iceberg/tests/namespace_create_test.rs +++ b/lib/crowdb-access-iceberg/tests/namespace_create_test.rs @@ -25,7 +25,11 @@ fn request(fixture: &TestNamespace, names: &[&str]) -> NamespaceCreateRequest { }, principal: "writer".into(), identifier: NamespaceIdentifier::new(names.iter().map(|name| (*name).into()).collect()).unwrap(), - properties: NamespaceProperties::new(BTreeMap::from([("owner".into(), "数据库".into())])).unwrap(), + properties: NamespaceProperties::new(BTreeMap::from([( + "owner".into(), + "\u{6570}\u{636e}\u{5e93}".into(), + )])) + .unwrap(), } } @@ -72,7 +76,7 @@ async fn native_create_publishes_complete_resources_and_retries_existing_names_a assert_eq!( serde_json::from_slice::(&outcome.body).unwrap(), serde_json::json!({ - "namespace": names, "properties": {"owner": "数据库"}, + "namespace": names, "properties": {"owner": "\u{6570}\u{636e}\u{5e93}"}, }) ); let authority = reader diff --git a/lib/crowdb-access-iceberg/tests/namespace_model_test.rs b/lib/crowdb-access-iceberg/tests/namespace_model_test.rs index fc5c8d506..9e2fdbd5d 100644 --- a/lib/crowdb-access-iceberg/tests/namespace_model_test.rs +++ b/lib/crowdb-access-iceberg/tests/namespace_model_test.rs @@ -8,7 +8,11 @@ use crowdb_access_iceberg::namespace::{ #[test] fn multipart_identifiers_preserve_unicode_and_delimiter_like_text() { - let components = vec!["数据库".into(), "a.b / %1F".into(), "leaf".into()]; + let components = vec![ + "\u{6570}\u{636e}\u{5e93}".into(), + "a.b / %1F".into(), + "leaf".into(), + ]; let identifier = NamespaceIdentifier::new(components.clone()).unwrap(); assert_eq!( NamespaceIdentifier::decode(&identifier.encode().unwrap()).unwrap(), @@ -105,7 +109,7 @@ fn property_limits_are_byte_based_and_include_result_cardinality() { for (key, value, valid) in [ ("k".repeat(1024), "v".repeat(8192), true), ("k".repeat(1025), String::new(), false), - ("键".repeat(342), String::new(), false), + ("\u{952e}".repeat(342), String::new(), false), ("k".into(), "v".repeat(8193), false), ("\0".into(), String::new(), false), ("k".into(), "\0".into(), false), diff --git a/lib/crowdb-access-iceberg/tests/namespace_record_test.rs b/lib/crowdb-access-iceberg/tests/namespace_record_test.rs index d68b890f4..609473594 100644 --- a/lib/crowdb-access-iceberg/tests/namespace_record_test.rs +++ b/lib/crowdb-access-iceberg/tests/namespace_record_test.rs @@ -22,7 +22,11 @@ fn authority() -> NamespaceAuthority { pending_operation: None, creating_child: None, rename: None, - properties: NamespaceProperties::new(BTreeMap::from([("owner".into(), "数据库".into())])).unwrap(), + properties: NamespaceProperties::new(BTreeMap::from([( + "owner".into(), + "\u{6570}\u{636e}\u{5e93}".into(), + )])) + .unwrap(), } } @@ -132,7 +136,7 @@ fn child_ranges_are_parent_scoped_and_separate_namespace_and_table_indexes() { Some(NamespaceId::from_bytes(&[255; 16]).unwrap()), ] { let range = child_range(catalog, parent, CatalogScope::NamespaceName).unwrap(); - for name in ["a", "longer", "数据库"] { + for name in ["a", "longer", "\u{6570}\u{636e}\u{5e93}"] { let key = name_key(catalog, parent, name).unwrap().encode().unwrap(); assert!(range.contains(&key)); assert!( diff --git a/lib/crowdb-access-iceberg/tests/namespace_repository_test.rs b/lib/crowdb-access-iceberg/tests/namespace_repository_test.rs index 05a4a669b..cd24d42a2 100644 --- a/lib/crowdb-access-iceberg/tests/namespace_repository_test.rs +++ b/lib/crowdb-access-iceberg/tests/namespace_repository_test.rs @@ -19,9 +19,9 @@ use fixture::TestNamespace; #[tokio::test] async fn lookup_walks_stable_parents_without_writing_or_creating_ancestors() { let fixture = TestNamespace::new().await; - let parent = fixture.authority(None, &["数据库"]); + let parent = fixture.authority(None, &["\u{6570}\u{636e}\u{5e93}"]); fixture.publish(&parent).await; - let child = fixture.authority(Some(parent.namespace), &["数据库", "child"]); + let child = fixture.authority(Some(parent.namespace), &["\u{6570}\u{636e}\u{5e93}", "child"]); fixture.publish(&child).await; let repository = NamespaceRepository::new(fixture.store.clone()); let writes = fixture.store.writes.load(Ordering::SeqCst); @@ -29,7 +29,11 @@ async fn lookup_walks_stable_parents_without_writing_or_creating_ancestors() { repository.load(fixture.context, &child.identifier).await.unwrap(), Some(child) ); - for names in [vec!["child"], vec!["absent", "child"], vec!["数据库", "absent"]] { + for names in [ + vec!["child"], + vec!["absent", "child"], + vec!["\u{6570}\u{636e}\u{5e93}", "absent"], + ] { let identifier = NamespaceIdentifier::new(names.into_iter().map(String::from).collect()).unwrap(); assert!(!repository.exists(fixture.context, &identifier).await.unwrap()); } diff --git a/lib/crowdb-access-iceberg/tests/namespace_update_test.rs b/lib/crowdb-access-iceberg/tests/namespace_update_test.rs index bad8adaa1..b8c003741 100644 --- a/lib/crowdb-access-iceberg/tests/namespace_update_test.rs +++ b/lib/crowdb-access-iceberg/tests/namespace_update_test.rs @@ -27,7 +27,7 @@ fn request(fixture: &TestNamespace, authority: &NamespaceAuthority) -> Namespace identifier: authority.identifier.clone(), changes: PropertyChanges { removals: vec!["old".into(), "absent".into()], - updates: BTreeMap::from([("owner".into(), "数据库".into())]), + updates: BTreeMap::from([("owner".into(), "\u{6570}\u{636e}\u{5e93}".into())]), }, } } @@ -66,7 +66,10 @@ async fn property_publication_is_atomic_and_does_not_change_identity_or_admissio assert_eq!(selected.pending_operation, None); assert_eq!( selected.properties.entries(), - &BTreeMap::from([("keep".into(), "value".into()), ("owner".into(), "数据库".into()),]) + &BTreeMap::from([ + ("keep".into(), "value".into()), + ("owner".into(), "\u{6570}\u{636e}\u{5e93}".into()), + ]) ); let before = fixture.store.writes.load(Ordering::Relaxed); let replay = repository.update_properties(&request).await.unwrap().unwrap(); @@ -109,14 +112,20 @@ async fn independent_requests_apply_changes_to_current_properties_without_a_ledg repository.update_properties(&request).await.unwrap(); request.changes.updates.insert("owner".into(), "second".into()); repository.update_properties(&request).await.unwrap(); - request.changes.updates.insert("owner".into(), "数据库".into()); + request + .changes + .updates + .insert("owner".into(), "\u{6570}\u{636e}\u{5e93}".into()); repository.update_properties(&request).await.unwrap(); let selected = repository .load(fixture.context, &authority.identifier) .await .unwrap() .unwrap(); - assert_eq!(selected.properties.entries().get("owner").unwrap(), "数据库"); + assert_eq!( + selected.properties.entries().get("owner").unwrap(), + "\u{6570}\u{636e}\u{5e93}" + ); assert_eq!(selected.property_revision, authority.property_revision + 3); assert!(selected.pending_operation.is_none()); } @@ -184,7 +193,10 @@ async fn concurrent_property_writers_rebase_only_after_a_definitive_cas_loss() { .unwrap() .unwrap(); assert_eq!(selected.property_revision, authority.property_revision + 2); - assert_eq!(selected.properties.entries().get("owner").unwrap(), "数据库"); + assert_eq!( + selected.properties.entries().get("owner").unwrap(), + "\u{6570}\u{636e}\u{5e93}" + ); assert_eq!(selected.properties.entries().get("second").unwrap(), "retained"); assert_eq!(selected.properties.entries().get("keep").unwrap(), "value"); assert_eq!(selected.pending_operation, None); @@ -342,7 +354,10 @@ async fn four_real_property_conflicts_merge_the_observed_authority_without_get() "retained" ); } - assert_eq!(selected.properties.entries().get("owner").unwrap(), "数据库"); + assert_eq!( + selected.properties.entries().get("owner").unwrap(), + "\u{6570}\u{636e}\u{5e93}" + ); assert_eq!(selected.properties.entries().get("keep").unwrap(), "value"); } diff --git a/lib/crowdb-access-iceberg/tests/native_storage_path_test.rs b/lib/crowdb-access-iceberg/tests/native_storage_path_test.rs index e6e2edb6c..6ea66d3cf 100644 --- a/lib/crowdb-access-iceberg/tests/native_storage_path_test.rs +++ b/lib/crowdb-access-iceberg/tests/native_storage_path_test.rs @@ -28,7 +28,14 @@ fn native_uri_preserves_immutable_identity_and_exact_filename() { assert_eq!(table.to_string().len(), StorageTablePath::PREFIX_BYTES); assert_eq!(table.to_string().parse::().unwrap(), table); assert_eq!(table.uri(), format!("s3://iceberg/{table}")); - for filename in ["metadata/v1.json", "数据/a+b", "a%2Fb", "a//b", "a/", "a b"] { + for filename in [ + "metadata/v1.json", + "\u{6570}\u{636e}/a+b", + "a%2Fb", + "a//b", + "a/", + "a b", + ] { let location = table.logical_file(filename).unwrap(); assert_eq!(location.table(), table); assert_eq!(location.relative_key(), filename); @@ -87,7 +94,7 @@ fn filename_limit_counts_three_ids_and_utf8_bytes() { location ); assert!(table.file(&(filename + "x")).is_err()); - assert!(table.file(&("x".repeat(available - 1) + "冰")).is_err()); + assert!(table.file(&("x".repeat(available - 1) + "\u{51b0}")).is_err()); } #[test] diff --git a/lib/crowdb-access-iceberg/tests/record_test.rs b/lib/crowdb-access-iceberg/tests/record_test.rs index 8760fd981..8acdd914d 100644 --- a/lib/crowdb-access-iceberg/tests/record_test.rs +++ b/lib/crowdb-access-iceberg/tests/record_test.rs @@ -103,7 +103,7 @@ fn every_root_phase_round_trips_with_its_exact_catalog_and_deadline() { #[test] fn authority_round_trip_binds_identity_to_its_key() { let catalog = CatalogId::random(); - let record = StorageRecord::Authority(CatalogAuthority::new(catalog, "目录".into()).unwrap()); + let record = StorageRecord::Authority(CatalogAuthority::new(catalog, "\u{76ee}\u{5f55}".into()).unwrap()); let encoded = record.encode().unwrap(); let key = IcebergKey::Catalog { catalog, diff --git a/lib/crowdb-access-iceberg/tests/table_metadata_defaults_test.rs b/lib/crowdb-access-iceberg/tests/table_metadata_defaults_test.rs index 89e424cbd..1509b4484 100644 --- a/lib/crowdb-access-iceberg/tests/table_metadata_defaults_test.rs +++ b/lib/crowdb-access-iceberg/tests/table_metadata_defaults_test.rs @@ -24,7 +24,7 @@ fn primitive_defaults_check_type_range_and_canonical_encodings() { ("double", json!(1e100), json!("NaN")), ("decimal(4,2)", json!("12.34"), json!("123.45")), ("decimal(4,2)", json!("1234e-2"), json!("12.3")), - ("string", json!("雪"), json!(5)), + ("string", json!("\u{96ea}"), json!(5)), ( "uuid", json!("12345678-1234-1234-1234-123456789abc"), diff --git a/lib/crowdb-access-iceberg/tests/table_metadata_document_test.rs b/lib/crowdb-access-iceberg/tests/table_metadata_document_test.rs index 051e9d790..05158daa6 100644 --- a/lib/crowdb-access-iceberg/tests/table_metadata_document_test.rs +++ b/lib/crowdb-access-iceberg/tests/table_metadata_document_test.rs @@ -35,7 +35,7 @@ fn official_java_metadata_roundtrips_without_rewriting() { fn version_envelopes_preserve_original_bytes_and_unknown_fields() { for version in 1..=3 { let mut value = fixture::metadata(version); - value["future-extension"] = json!({"nested":[null,true,5,"雪"]}); + value["future-extension"] = json!({"nested":[null,true,5,"\u{96ea}"]}); value["location"] = json!(fixture::table().to_string().trim_end_matches('/')); let bytes = format!(" \n{}\n ", serde_json::to_string_pretty(&value).unwrap()).into_bytes(); let head = fixture::head( diff --git a/lib/crowdb-access-s3/src/bucket.rs b/lib/crowdb-access-s3/src/bucket.rs index c489fdd12..83406047f 100644 --- a/lib/crowdb-access-s3/src/bucket.rs +++ b/lib/crowdb-access-s3/src/bucket.rs @@ -3,12 +3,13 @@ //! Durable bucket-name lifecycle on the deliberately slow control path. +mod listed_record; mod listing; pub use listing::{BucketListPage, BucketListRequest, BucketListSigner}; use crate::metadata::{ BucketId, BucketNameRecord, ChunkKvMetadataStore, MetadataKey, MetadataRecordError, MetadataStoreError, - ObjectRecord, PutIfAbsentOutcome, TenantId, + PutIfAbsentOutcome, TenantId, }; const LIST_MAX_BYTES: usize = 4 * 1024 * 1024; @@ -132,15 +133,16 @@ pub async fn list_buckets( .await?; values .into_iter() - // Object keys can share this legacy interval when their random bucket - // ID starts with a low byte. Decode them as objects before treating a - // failed bucket decode as corrupt bucket metadata. - .filter_map(|value| match BucketNameRecord::decode(&value.value) { - Ok(record) if !record.tombstone => Some(Ok(record)), - Ok(_) => None, - Err(_) if ObjectRecord::decode(&value.value).is_ok() => None, - Err(error) => Some(Err(error.into())), - }) + // Resource keys can share this legacy interval when their random bucket + // ID starts with a low byte. Validate their complete persisted identities + // before treating a row as corrupt bucket metadata. + .filter_map( + |value| match listed_record::decode_bucket_row(tenant, &value.key, &value.value) { + Ok(Some(record)) if !record.tombstone => Some(Ok(record)), + Ok(_) => None, + Err(error) => Some(Err(error.into())), + }, + ) .collect() } diff --git a/lib/crowdb-access-s3/src/bucket/listed_record.rs b/lib/crowdb-access-s3/src/bucket/listed_record.rs new file mode 100644 index 000000000..c56473a9a --- /dev/null +++ b/lib/crowdb-access-s3/src/bucket/listed_record.rs @@ -0,0 +1,47 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Classify rows in the legacy bucket interval by complete persisted identity. + +use crate::metadata::{ + BucketNameRecord, MetadataKey, MetadataRecordError, MultipartPartRecord, MultipartSessionRecord, + ObjectRecord, TenantId, +}; + +pub(super) fn decode_bucket_row( + tenant: &TenantId, + key: &[u8], + value: &[u8], +) -> Result, MetadataRecordError> { + let error = match BucketNameRecord::decode(value) { + Ok(record) => { + if record.tenant == *tenant + && MetadataKey::bucket_name(tenant, &record.name).is_ok_and(|expected| expected == key) + { + return Ok(Some(record)); + } + MetadataRecordError::Invalid + } + Err(error) => error, + }; + if ObjectRecord::decode(value).is_ok_and(|record| { + MetadataKey::object(tenant, record.bucket_id, &record.key).is_ok_and(|expected| expected == key) + }) || MultipartSessionRecord::decode_unbound(value).is_ok_and(|record| { + MetadataKey::multipart_session(tenant, record.bucket_id, &record.upload_id) == key + }) || MultipartPartRecord::decode_unbound(value).is_ok_and(|record| { + MetadataKey::multipart_part(tenant, record.bucket_id, &record.upload_id, record.number) + .is_ok_and(|expected| expected == key) + || MetadataKey::multipart_part_generation( + tenant, + record.bucket_id, + &record.upload_id, + record.number, + record.revision, + ) + .is_ok_and(|expected| expected == key) + }) || MetadataKey::matches_multipart_index(tenant, key, value) + { + return Ok(None); + } + Err(error) +} diff --git a/lib/crowdb-access-s3/src/bucket/listing.rs b/lib/crowdb-access-s3/src/bucket/listing.rs index 47059c299..02a79ba21 100644 --- a/lib/crowdb-access-s3/src/bucket/listing.rs +++ b/lib/crowdb-access-s3/src/bucket/listing.rs @@ -8,10 +8,8 @@ use hmac::{Hmac, Mac}; use serde::{Deserialize, Serialize}; use sha2::Sha256; -use super::BucketError; -use crate::metadata::{ - BucketNameRecord, ChunkKvMetadataStore, MetadataKey, MetadataStoreError, ObjectRecord, TenantId, -}; +use super::{listed_record::decode_bucket_row, BucketError}; +use crate::metadata::{BucketNameRecord, ChunkKvMetadataStore, MetadataKey, MetadataStoreError, TenantId}; const MAX_CURSOR_BYTES: usize = 64 * 1024; const CURSOR_CONTEXT: &[u8] = b"crowdb-list-buckets-v1\0"; @@ -87,20 +85,13 @@ impl BucketListSigner { let mut buckets = Vec::new(); let mut continuation = None; for (index, value) in page.items.into_iter().enumerate() { - match BucketNameRecord::decode(&value.value) { - Ok(record) => { - if record.tenant != *tenant - || MetadataKey::bucket_name(tenant, &record.name)? != value.key - || !record.name.starts_with(request.prefix) - { - return Err(crate::metadata::MetadataRecordError::Invalid.into()); - } - if !record.tombstone { - buckets.push(record); - } + if let Some(record) = decode_bucket_row(tenant, &value.key, &value.value)? { + if !record.name.starts_with(request.prefix) { + return Err(crate::metadata::MetadataRecordError::Invalid.into()); + } + if !record.tombstone { + buckets.push(record); } - Err(_) if ObjectRecord::decode(&value.value).is_ok() => {} - Err(error) => return Err(error.into()), } if buckets.len() == request.max_buckets || index + 1 == count { if index + 1 < count || page.continuation.is_some() { diff --git a/lib/crowdb-access-s3/src/metadata/key.rs b/lib/crowdb-access-s3/src/metadata/key.rs index 051213c21..91dd04420 100644 --- a/lib/crowdb-access-s3/src/metadata/key.rs +++ b/lib/crowdb-access-s3/src/metadata/key.rs @@ -341,6 +341,38 @@ impl MetadataKey { key.extend_from_slice(&revision.to_be_bytes()); Ok(key) } + + pub(crate) fn matches_multipart_index(tenant: &TenantId, key: &[u8], value: &[u8]) -> bool { + let namespace = namespace_prefix(tenant); + let Some(pointer) = value.strip_prefix(namespace.as_slice()) else { + return false; + }; + if pointer.len() != 34 { + return false; + } + let mut bucket = [0; 16]; + bucket.copy_from_slice(&pointer[..16]); + let bucket = BucketId::new(bucket); + let mut upload = [0; 16]; + upload.copy_from_slice(&pointer[17..33]); + if upload == [0; 16] || value != Self::multipart_session(tenant, bucket, &upload) { + return false; + } + let prefix = Self::multipart_session_prefix(tenant, bucket); + let Some(encoded) = key + .strip_prefix(prefix.as_slice()) + .and_then(|key| key.strip_suffix(&upload)) + else { + return false; + }; + if encoded.len() > MAX_KEY_BYTES * 2 + 2 { + return false; + } + let Some(object) = decode_ordered_component(encoded) else { + return false; + }; + Self::multipart_upload_index(tenant, bucket, &object, &upload).is_ok_and(|expected| expected == key) + } } #[derive(Clone, Debug, Eq, PartialEq)] @@ -392,6 +424,25 @@ fn append_ordered_bytes_prefix(output: &mut Vec, value: &[u8]) { } } +fn decode_ordered_component(mut encoded: &[u8]) -> Option> { + let mut decoded = Vec::new(); + while let Some((&first, remaining)) = encoded.split_first() { + encoded = remaining; + if first != 0 { + decoded.push(first); + continue; + } + let (&escape, remaining) = encoded.split_first()?; + match escape { + 0 => return remaining.is_empty().then_some(decoded), + 0xff => decoded.push(0), + _ => return None, + } + encoded = remaining; + } + None +} + fn increment_lexicographic(value: &mut Vec) { while let Some(byte) = value.pop() { if byte != u8::MAX { diff --git a/lib/crowdb-access-s3/src/metadata/multipart.rs b/lib/crowdb-access-s3/src/metadata/multipart.rs index c7b795173..32378161b 100644 --- a/lib/crowdb-access-s3/src/metadata/multipart.rs +++ b/lib/crowdb-access-s3/src/metadata/multipart.rs @@ -240,14 +240,19 @@ impl MultipartPartRecord { upload_id: &[u8; 16], number: u16, ) -> Result { - let record: Self = decode(PART_MAGIC, bytes)?; - record.validate()?; + let record = Self::decode_unbound(bytes)?; if record.bucket_id != bucket || &record.upload_id != upload_id || record.number != number { return Err(MultipartRecordError::Identity); } Ok(record) } + pub(crate) fn decode_unbound(bytes: &[u8]) -> Result { + let record: Self = decode(PART_MAGIC, bytes)?; + record.validate()?; + Ok(record) + } + fn validate(&self) -> Result<(), MultipartRecordError> { if self.upload_id == [0; 16] || self.number == 0 || self.number > 10_000 || self.revision == 0 { return Err(MultipartRecordError::Invalid); diff --git a/lib/crowdb-access-s3/tests/bucket_listing_collision_test.rs b/lib/crowdb-access-s3/tests/bucket_listing_collision_test.rs new file mode 100644 index 000000000..fca8176a4 --- /dev/null +++ b/lib/crowdb-access-s3/tests/bucket_listing_collision_test.rs @@ -0,0 +1,196 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/bucket_client.rs"] +mod bucket_client; +#[path = "common/bucket_listing_collision.rs"] +mod fixture; + +use crowdb_access_s3::{ + bucket::{self, BucketError, BucketListRequest, BucketListSigner}, + metadata::{BucketId, MetadataKey, ObjectRecord, TenantId}, +}; +use fixture::{multipart_rows, paginated, session, store}; + +#[tokio::test] +async fn simple_listing_skips_valid_multipart_records_sharing_the_bucket_interval() { + let tenant = TenantId::new(b"tenant".to_vec()).unwrap(); + let mut failures = Vec::new(); + for (kind, key, value) in multipart_rows(&tenant) { + assert!( + key >= MetadataKey::bucket_name_prefix(&tenant) && key < MetadataKey::bucket_name_end(&tenant) + ); + let store = store(&tenant, key, value).await; + match bucket::list_buckets(&store, &tenant, 100).await { + Ok(buckets) => assert_eq!( + buckets.iter().map(|b| b.name.as_slice()).collect::>(), + vec![b"expected"] + ), + Err(error) => failures.push(format!("{kind}: {error:?}")), + } + } + assert!( + failures.is_empty(), + "valid multipart resource poisoned simple bucket listing: {failures:?}" + ); +} + +#[tokio::test] +async fn paginated_listing_skips_valid_multipart_records_sharing_the_bucket_interval() { + let tenant = TenantId::new(b"tenant".to_vec()).unwrap(); + let mut failures = Vec::new(); + for (kind, key, value) in multipart_rows(&tenant) { + let store = store(&tenant, key, value).await; + match paginated(&store, &tenant).await { + Ok(names) => assert_eq!(names, vec![b"expected".to_vec()]), + Err(error) => failures.push(format!("{kind}: {error:?}")), + } + } + assert!( + failures.is_empty(), + "valid multipart resource poisoned paginated bucket listing: {failures:?}" + ); +} + +#[tokio::test] +async fn malformed_bucket_rows_are_not_hidden_as_multipart_resources() { + let tenant = TenantId::new(b"tenant".to_vec()).unwrap(); + for value in [b"corrupt-bucket".to_vec(), session().encode().unwrap()] { + let key = MetadataKey::bucket_name(&tenant, b"bad-bucket").unwrap(); + let store = store(&tenant, key, value).await; + assert!(matches!( + bucket::list_buckets(&store, &tenant, 100).await, + Err(BucketError::Record(_)) + )); + assert!(matches!( + paginated(&store, &tenant).await, + Err(BucketError::Record(_)) + )); + } +} + +#[tokio::test] +async fn legacy_objects_are_skipped_only_under_their_exact_key() { + let tenant = TenantId::new(b"tenant".to_vec()).unwrap(); + let record = ObjectRecord { + bucket_id: session().bucket_id, + key: b"legacy-object".to_vec(), + logical_length: 1, + checksum: vec![9; 16], + etag: "etag".into(), + created_at_ms: 100, + modified_at_ms: 100, + content_type: "text/plain".into(), + attributes: Vec::new(), + data_reference: vec![7], + data_length: 1, + }; + let bytes = record.encode().unwrap(); + let good = store( + &tenant, + MetadataKey::object(&tenant, record.bucket_id, &record.key).unwrap(), + bytes.clone(), + ) + .await; + assert_eq!(bucket::list_buckets(&good, &tenant, 100).await.unwrap().len(), 1); + assert_eq!( + paginated(&good, &tenant).await.unwrap(), + vec![b"expected".to_vec()] + ); + let bad = store( + &tenant, + MetadataKey::bucket_name(&tenant, b"bad-object").unwrap(), + bytes, + ) + .await; + assert!(matches!( + bucket::list_buckets(&bad, &tenant, 100).await, + Err(BucketError::Record(_)) + )); + assert!(matches!( + paginated(&bad, &tenant).await, + Err(BucketError::Record(_)) + )); +} + +#[tokio::test] +async fn foreign_only_page_keeps_its_cursor_and_returns_later_buckets_once() { + let tenant = TenantId::new(b"tenant".to_vec()).unwrap(); + let (_, key, value) = multipart_rows(&tenant).remove(0); + let store = store(&tenant, key, value).await; + let signer = BucketListSigner::new(b"secret".to_vec()).unwrap(); + let options = |continuation| BucketListRequest { + prefix: b"", + continuation, + max_buckets: 1, + max_scan_items: 1, + max_scan_bytes: 65536, + now: 150, + ttl: 30, + }; + let first = signer.list(&store, &tenant, &options(None)).await.unwrap(); + assert!(first.buckets.is_empty()); + assert!(first.continuation.is_some()); + let second = signer + .list(&store, &tenant, &options(first.continuation.as_deref())) + .await + .unwrap(); + assert_eq!( + second + .buckets + .iter() + .map(|b| b.name.as_slice()) + .collect::>(), + vec![b"expected"] + ); + assert!(second.continuation.is_none()); +} + +#[tokio::test] +async fn multipart_rows_with_foreign_or_corrupt_identities_remain_errors() { + let tenant = TenantId::new(b"tenant".to_vec()).unwrap(); + let other = TenantId::new(b"other-tenant".to_vec()).unwrap(); + let mut bad = Vec::new(); + for (kind, key, value) in multipart_rows(&tenant) { + let mut wrong_value = value.clone(); + wrong_value[0] ^= 0x40; + bad.push((format!("{kind} unknown format"), key.clone(), wrong_value)); + let mut wrong_key = key.clone(); + let last = wrong_key.len() - 1; + wrong_key[last] ^= 1; + bad.push((format!("{kind} foreign key"), wrong_key, value)); + } + let original = session(); + let mut foreign = original.clone(); + foreign.bucket_id = BucketId::new([3; 16]); + bad.push(( + "foreign bucket session".into(), + MetadataKey::multipart_session(&tenant, original.bucket_id, &original.upload_id), + foreign.encode().unwrap(), + )); + let (_, index_key, index_value) = multipart_rows(&tenant).pop().unwrap(); + let foreign_pointer = MetadataKey::multipart_session(&other, original.bucket_id, &original.upload_id); + bad.push(("foreign tenant index".into(), index_key, foreign_pointer)); + let mut malformed_index = MetadataKey::multipart_session_prefix(&tenant, original.bucket_id); + malformed_index.extend_from_slice(b"object\0\x01"); + malformed_index.extend_from_slice(&original.upload_id); + bad.push(( + "invalid ordered index component".into(), + malformed_index, + index_value, + )); + for (kind, key, value) in bad { + let store = store(&tenant, key, value).await; + assert!( + matches!( + bucket::list_buckets(&store, &tenant, 100).await, + Err(BucketError::Record(_)) + ), + "simple listing accepted {kind}" + ); + assert!( + matches!(paginated(&store, &tenant).await, Err(BucketError::Record(_))), + "paginated listing accepted {kind}" + ); + } +} diff --git a/lib/crowdb-access-s3/tests/common/bucket_listing_collision.rs b/lib/crowdb-access-s3/tests/common/bucket_listing_collision.rs new file mode 100644 index 000000000..aecb5f7f6 --- /dev/null +++ b/lib/crowdb-access-s3/tests/common/bucket_listing_collision.rs @@ -0,0 +1,197 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::bucket_client; + +use async_trait::async_trait; +use crowdb_access_s3::{ + bucket::{BucketError, BucketListRequest, BucketListSigner}, + metadata::{ + BucketId, BucketNameRecord, ChunkKvMetadataStore, MetadataKey, MultipartPartRecord, MultipartPhase, + MultipartSessionRecord, TenantId, + }, +}; +use crowdb_chunk_kv_client::{ChunkKvTransport, Result}; +use crowdb_protocol::{chunk_kv::*, chunkdb::rpc::Location}; +use std::sync::Arc; + +struct TestTransport(Vec); +type TestMetadataRow = (&'static str, Vec, Vec); + +#[async_trait] +impl ChunkKvTransport for TestTransport { + async fn point(&self, _: &str, _: &PointRequest) -> Result { + panic!("listing must only scan"); + } + + async fn seek(&self, _: &str, _: &SeekRequest) -> Result { + panic!("listing must only scan"); + } + + async fn scan(&self, _: &str, request: &ScanRequest) -> Result { + let mut items: Vec<_> = self + .0 + .iter() + .filter(|item| { + request.start.as_ref().map_or(true, |start| item.key >= *start) + && request.end.as_ref().map_or(true, |end| item.key < *end) + && request + .continuation + .as_ref() + .map_or(true, |cursor| item.key > cursor.last_key) + }) + .cloned() + .collect(); + let more = items.len() > request.limit as usize; + items.truncate(request.limit as usize); + let continuation = more.then(|| ScanContinuation { + direction: request.direction, + last_key: items.last().unwrap().key.clone(), + partition_id: request.routing.partition_id, + owner_epoch: request.routing.owner_epoch, + map_revision: request.routing.map_revision, + }); + Ok(ChunkKvResponse { + map_revision: 1, + journal_position: None, + result: Ok(OperationResult::Scan { items, continuation }), + }) + } +} + +pub(super) fn session() -> MultipartSessionRecord { + let mut bucket = [0x11; 16]; + bucket[0] = 1; + MultipartSessionRecord { + bucket_id: BucketId::new(bucket), + object_key: b"multipart\0object".to_vec(), + upload_id: [7; 16], + revision: 1, + phase: MultipartPhase::Open, + created_ms: 100, + expires_ms: 200, + content_type: "application/octet-stream".into(), + attributes: Vec::new(), + max_parts: 10, + max_part_bytes: 100, + max_object_bytes: 500, + max_staged_bytes: 1000, + part_count: 0, + staged_bytes: 0, + pending: None, + selection: None, + legacy_observation: None, + } +} + +pub(super) fn multipart_rows(tenant: &TenantId) -> Vec { + let session = session(); + let session_key = MetadataKey::multipart_session(tenant, session.bucket_id, &session.upload_id); + let session_bytes = session.encode().unwrap(); + assert_eq!( + MultipartSessionRecord::decode( + &session_bytes, + session.bucket_id, + &session.object_key, + &session.upload_id + ) + .unwrap(), + session + ); + let part = MultipartPartRecord { + bucket_id: session.bucket_id, + upload_id: session.upload_id, + number: 1, + revision: 1, + modified_ms: 150, + length: 5, + raw_md5: [9; 16], + locations: vec![Location { + chunk_id: Some(crowdb_protocol::common::ChunkId { high: 1, low: 2 }), + offset: 10, + length: 39, + logical_offset: 0, + logical_length: 5, + }], + }; + let part_bytes = part.encode().unwrap(); + assert_eq!( + MultipartPartRecord::decode(&part_bytes, part.bucket_id, &part.upload_id, 1).unwrap(), + part + ); + vec![ + ("session", session_key.clone(), session_bytes), + ( + "part", + MetadataKey::multipart_part(tenant, part.bucket_id, &part.upload_id, 1).unwrap(), + part_bytes.clone(), + ), + ( + "part-generation", + MetadataKey::multipart_part_generation(tenant, part.bucket_id, &part.upload_id, 1, 1).unwrap(), + part_bytes, + ), + ( + "upload-index", + MetadataKey::multipart_upload_index( + tenant, + session.bucket_id, + &session.object_key, + &session.upload_id, + ) + .unwrap(), + session_key, + ), + ] +} + +pub(super) async fn store( + tenant: &TenantId, + extra_key: Vec, + extra_value: Vec, +) -> ChunkKvMetadataStore { + let mapping = BucketNameRecord { + tenant: tenant.clone(), + name: b"expected".to_vec(), + bucket_id: session().bucket_id, + tombstone: false, + }; + let mut items = vec![ + RpcValue { + key: MetadataKey::bucket_name(tenant, &mapping.name).unwrap(), + value: mapping.encode(), + revision: 1, + }, + RpcValue { + key: extra_key, + value: extra_value, + revision: 2, + }, + ]; + items.sort_by(|a, b| a.key.cmp(&b.key)); + let client = Arc::new(bucket_client::client(Arc::new(TestTransport(items)))); + client.refresh_catalog().await.unwrap(); + ChunkKvMetadataStore::new(client) +} + +pub(super) async fn paginated( + store: &ChunkKvMetadataStore, + tenant: &TenantId, +) -> std::result::Result>, BucketError> { + BucketListSigner::new(b"secret".to_vec())? + .list( + store, + tenant, + &BucketListRequest { + prefix: b"", + continuation: None, + max_buckets: 100, + max_scan_items: 100, + max_scan_bytes: 65536, + now: 150, + ttl: 30, + }, + ) + .await + .map(|page| page.buckets.into_iter().map(|bucket| bucket.name).collect()) +} diff --git a/lib/crowdb-access-s3/tests/copy_test.rs b/lib/crowdb-access-s3/tests/copy_test.rs index 2a86f8165..57e23dad8 100644 --- a/lib/crowdb-access-s3/tests/copy_test.rs +++ b/lib/crowdb-access-s3/tests/copy_test.rs @@ -76,7 +76,7 @@ fn conditions_limits_and_self_copy_use_the_captured_record() { fn source_addresses_decode_once_and_reject_ambiguous_selectors() { assert_eq!( copy::source("/bucket/a%2Fb%20%25%2B%E9%9B%AA").unwrap(), - (b"bucket".to_vec(), "a/b %+雪".as_bytes().to_vec()) + (b"bucket".to_vec(), "a/b %+\u{96ea}".as_bytes().to_vec()) ); assert_eq!( copy::source("bucket/question%3FversionId%3Dliteral").unwrap().1, diff --git a/lib/crowdb-access-s3/tests/delete_test.rs b/lib/crowdb-access-s3/tests/delete_test.rs index eff7b43c1..7ddb0797c 100644 --- a/lib/crowdb-access-s3/tests/delete_test.rs +++ b/lib/crowdb-access-s3/tests/delete_test.rs @@ -57,9 +57,12 @@ fn batch_route_and_exact_keys() { ) .unwrap(); assert_eq!(route.operation, S3Operation::DeleteObjects); - let selection = DeleteSelection::parse("a&<雪a&<雪true".as_bytes()).unwrap(); + let selection = DeleteSelection::parse("a&<\u{96ea}a&<\u{96ea}true".as_bytes()).unwrap(); assert!(selection.quiet); - assert_eq!(selection.keys, vec!["a&<雪".as_bytes(), "a&<雪".as_bytes()]); + assert_eq!( + selection.keys, + vec!["a&<\u{96ea}".as_bytes(), "a&<\u{96ea}".as_bytes()] + ); } #[test] diff --git a/lib/crowdb-access-s3/tests/object_inspection_test.rs b/lib/crowdb-access-s3/tests/object_inspection_test.rs index a8c6d9e6e..b0f7cde4c 100644 --- a/lib/crowdb-access-s3/tests/object_inspection_test.rs +++ b/lib/crowdb-access-s3/tests/object_inspection_test.rs @@ -10,7 +10,7 @@ fn record(locations: &[Location]) -> ObjectRecord { let length = locations.iter().map(|location| location.logical_length).sum(); ObjectRecord { bucket_id: BucketId::new([7; 16]), - key: "folder/中文.bin".as_bytes().to_vec(), + key: "folder/\u{4e2d}\u{6587}.bin".as_bytes().to_vec(), logical_length: length, checksum: vec![1; 16], etag: "etag".into(), diff --git a/lib/crowdb-access-s3/tests/user_metadata_test.rs b/lib/crowdb-access-s3/tests/user_metadata_test.rs index f933a4c0e..baa13f86c 100644 --- a/lib/crowdb-access-s3/tests/user_metadata_test.rs +++ b/lib/crowdb-access-s3/tests/user_metadata_test.rs @@ -40,7 +40,7 @@ fn metadata_enforces_exact_combined_size_and_unique_valid_headers() { for (name, value) in [ ("x-amz-meta-", "empty-name"), ("x-amz-meta-a", "tab\tvalue"), - ("x-amz-meta-a", "雪"), + ("x-amz-meta-a", "\u{96ea}"), ] { input.clear(); input.insert(name, HeaderValue::from_bytes(value.as_bytes()).unwrap()); diff --git a/lib/crowdb-chunk-client/src/chunk/chunk_reader.rs b/lib/crowdb-chunk-client/src/chunk/chunk_reader.rs index 62fdce7ba..5bd9787bf 100644 --- a/lib/crowdb-chunk-client/src/chunk/chunk_reader.rs +++ b/lib/crowdb-chunk-client/src/chunk/chunk_reader.rs @@ -23,6 +23,7 @@ mod frame_views; mod framing; mod layout; mod layout_cache; +mod locality; mod location; mod stream; mod verified_frame; @@ -37,6 +38,8 @@ const DEFAULT_STREAM_WINDOW: usize = 1024 * 1024; /// Bounded retry, streaming, and EC-recovery memory policy. #[derive(Debug, Clone)] pub struct ChunkReadPolicy { + /// Prefer this node, then its rack, when retained routing covers a mirror. + pub preferred_node_id: Option, pub stream_window_bytes: usize, pub stream_slots: usize, pub global_stream_bytes: usize, @@ -92,6 +95,7 @@ impl LayoutSnapshot { impl Default for ChunkReadPolicy { fn default() -> Self { Self { + preferred_node_id: None, stream_window_bytes: DEFAULT_STREAM_WINDOW, stream_slots: 3, global_stream_bytes: 256 * 1024 * 1024, @@ -204,9 +208,10 @@ impl ChunkReader { let deadline = started + Duration::from_millis(response.layout_validity_ms) .saturating_sub(self.policy.layout_safety_margin); - let chunk = response + let mut chunk = response .chunk .ok_or_else(|| ReadError::ChunkDeleted(format!("{}:{}", chunk_id.high, chunk_id.low)))?; + self.prefer_local_mirrors(&mut chunk); Ok(LayoutSnapshot { chunk_id, chunk: Arc::new(chunk), diff --git a/lib/crowdb-chunk-client/src/chunk/chunk_reader/locality.rs b/lib/crowdb-chunk-client/src/chunk/chunk_reader/locality.rs new file mode 100644 index 000000000..c92cb56c3 --- /dev/null +++ b/lib/crowdb-chunk-client/src/chunk/chunk_reader/locality.rs @@ -0,0 +1,57 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Retained topology orders equivalent mirror reads without granting authority. + +use super::ChunkReader; +use crowdb_protocol::chunkdb::rpc::{Chunk, Strip}; +use std::collections::HashMap; + +impl ChunkReader { + pub(super) fn prefer_local_mirrors(&self, chunk: &mut Chunk) { + let Some(node) = self.policy.preferred_node_id else { + return; + }; + let Some(topology) = self.strip_reader.disk_topology() else { + return; + }; + let placements: HashMap<_, _> = topology + .disks() + .map(|disk| { + ( + (disk.disk_id.high, disk.disk_id.low), + (disk.rack_id, disk.node_id), + ) + }) + .collect(); + let Some(rack) = placements + .values() + .find_map(|(rack, owner)| (*owner == node).then_some(*rack)) + else { + return; + }; + if placements + .values() + .any(|(owner_rack, owner_node)| *owner_node == node && *owner_rack != rack) + { + return; + } + for strip in &mut chunk.strips { + let Some(Strip::MirrorStrip(mirror)) = &mut strip.strip else { + continue; + }; + if !mirror.segments.iter().all(|segment| { + segment + .disk_id + .is_some_and(|id| placements.contains_key(&(id.high, id.low))) + }) { + continue; + } + mirror.segments.sort_by_key(|segment| { + let id = segment.disk_id.expect("complete mirror placement hints"); + let (owner_rack, owner_node) = placements[&(id.high, id.low)]; + (owner_node != node, owner_rack != rack) + }); + } + } +} diff --git a/lib/crowdb-chunk-client/src/chunk/frame_observation/mirrored.rs b/lib/crowdb-chunk-client/src/chunk/frame_observation/mirrored.rs index 22fb4183a..db33f5218 100644 --- a/lib/crowdb-chunk-client/src/chunk/frame_observation/mirrored.rs +++ b/lib/crowdb-chunk-client/src/chunk/frame_observation/mirrored.rs @@ -17,6 +17,7 @@ use crate::{DiskWriter, STREAM_CHUNK_BYTES}; mod read; use read::MirroredWindowReader; +mod order; struct MirroredLayout { chunk: Arc, @@ -59,6 +60,7 @@ impl MirroredFrameReaders { return Err(invalid_layout("chunk purpose, owner, epoch or state is invalid")); } let (capacity, replicas) = validate_layout(&chunk, chunk_id)?; + let chunk = order::by_rack(chunk, disk_io.as_ref()); let layout = Arc::new(MirroredLayout { chunk, disk_io, diff --git a/lib/crowdb-chunk-client/src/chunk/frame_observation/mirrored/order.rs b/lib/crowdb-chunk-client/src/chunk/frame_observation/mirrored/order.rs new file mode 100644 index 000000000..3d26a93fe --- /dev/null +++ b/lib/crowdb-chunk-client/src/chunk/frame_observation/mirrored/order.rs @@ -0,0 +1,47 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Keep each observation lane in a consistent failure domain when possible. + +use crate::DiskWriter; +use crowdb_protocol::chunkdb::rpc::{Chunk, Strip}; +use std::{collections::HashMap, sync::Arc}; + +pub(super) fn by_rack(mut chunk: Arc, disk_io: &dyn DiskWriter) -> Arc { + let Some(topology) = disk_io.disk_topology() else { + return chunk; + }; + let placements: HashMap<_, _> = topology + .disks() + .map(|disk| { + ( + (disk.disk_id.high, disk.disk_id.low), + (disk.rack_id, disk.node_id, disk.disk_group_id), + ) + }) + .collect(); + // Retained routing is an ordering hint, never authority to add or omit a + // replica. Incomplete hints preserve the supplied layout's original order. + let complete = chunk.strips.iter().all(|strip| match &strip.strip { + Some(Strip::MirrorStrip(mirror)) => mirror.segments.iter().all(|segment| { + segment + .disk_id + .is_some_and(|id| placements.contains_key(&(id.high, id.low))) + }), + _ => false, + }); + if !complete { + return chunk; + } + for strip in &mut Arc::make_mut(&mut chunk).strips { + if let Some(Strip::MirrorStrip(mirror)) = &mut strip.strip { + mirror.segments.sort_by_key(|segment| { + segment + .disk_id + .and_then(|id| placements.get(&(id.high, id.low))) + .copied() + }); + } + } + chunk +} diff --git a/lib/crowdb-chunk-client/src/chunk/strip_reader.rs b/lib/crowdb-chunk-client/src/chunk/strip_reader.rs index 91de3cc04..f583e88f0 100644 --- a/lib/crowdb-chunk-client/src/chunk/strip_reader.rs +++ b/lib/crowdb-chunk-client/src/chunk/strip_reader.rs @@ -35,6 +35,10 @@ pub struct StripReader { } impl StripReader { + pub(crate) fn disk_topology(&self) -> Option { + self.disk_io.disk_topology() + } + pub fn new( disk_io: Arc, recovery_memory: Arc, diff --git a/lib/crowdb-chunk-client/src/client.rs b/lib/crowdb-chunk-client/src/client.rs index 047797711..638b33b13 100644 --- a/lib/crowdb-chunk-client/src/client.rs +++ b/lib/crowdb-chunk-client/src/client.rs @@ -8,7 +8,7 @@ use disk_metrics::MetricsDiskWriter; mod readiness; #[cfg(feature = "test-util")] -pub use readiness::observe_readiness_phase_for_tests; +pub use readiness::{observe_disk_readiness_for_tests, observe_readiness_phase_for_tests}; mod connection; mod observation; mod writer_policy; diff --git a/lib/crowdb-chunk-client/src/client/disk_metrics.rs b/lib/crowdb-chunk-client/src/client/disk_metrics.rs index 3fbc412ed..af89b5b4f 100644 --- a/lib/crowdb-chunk-client/src/client/disk_metrics.rs +++ b/lib/crowdb-chunk-client/src/client/disk_metrics.rs @@ -242,6 +242,27 @@ impl DiskWriter for MetricsDiskWriter { result } + async fn read_priority( + &self, + seg: &Segment, + unit_bytes: u64, + segment_offset: u64, + length: u32, + ) -> Result { + let mut operation = self.metrics.diskio_read.start(); + let result = self + .inner + .read_priority(seg, unit_bytes, segment_offset, length) + .await; + if let Ok(data) = &result { + self.metrics + .diskio_read_bytes + .observe(u64::try_from(data.len()).unwrap_or(u64::MAX)); + operation.mark_success(); + } + result + } + async fn write_at_byte_offset( &self, seg: &Segment, diff --git a/lib/crowdb-chunk-client/src/client/readiness.rs b/lib/crowdb-chunk-client/src/client/readiness.rs index 04b550889..239ab1ef1 100644 --- a/lib/crowdb-chunk-client/src/client/readiness.rs +++ b/lib/crowdb-chunk-client/src/client/readiness.rs @@ -3,8 +3,12 @@ //! Read-only ChunkDB and physical DiskIO evidence through current client routes. -use crowdb_protocol::{common::HwStatus, diskdb::rpc::Segment}; -use std::{future::Future, time::Duration}; +use crowdb_kv_client::DiskRecord; +use crowdb_protocol::{ + common::{DiskId, HwStatus}, + diskdb::rpc::Segment, +}; +use std::{future::Future, sync::Arc, time::Duration}; use tokio::time::Instant; use crate::metrics::{OperationMetrics, ReadinessMetricsSnapshot}; @@ -24,61 +28,27 @@ impl ChunkIoClient { .topology .as_ref() .ok_or_else(|| IoError::Topology("readiness requires authoritative discovery".into()))?; - let (disks, chunks) = tokio::join!( - observe("hardware", deadline, &self.readiness.hardware, async { + let online = observe_dependencies( + deadline, + &self.readiness, + topology.disk_writer.clone(), + async { topology .hardware .list_all_disks() .await .map_err(|error| IoError::Topology(error.to_string())) - }), - observe("ChunkDB", deadline, &self.readiness.chunkdb, async { + }, + async { Box::pin(topology.chunkdb.observe_readiness()) .await .map_err(|error| IoError::ReadFailed(format!("ChunkDB read: {error}"))) - }), - ); - chunks?; - let disks = disks?; - let online: Vec<_> = disks - .iter() - .filter(|disk| disk.value.status == HwStatus::Up as i32) - .collect(); - if online.is_empty() { - return Err(IoError::Topology("no online disks are available".into())); - } - // Keep background observation work bounded independently of cluster size. - for batch in online.chunks(8) { - let mut reads = tokio::task::JoinSet::new(); - for disk in batch { - let writer = topology.disk_writer.clone(); - let metrics = self.readiness.clone(); - let id = disk.disk_id; - reads.spawn(async move { - let segment = Segment { - disk_id: Some(id), - unit_count: 1, - ..Default::default() - }; - observe("DiskIO", deadline, &metrics.diskio, async { - let bytes = writer.read(&segment, 4096, 0, 4096).await?; - if bytes.len() != 4096 { - return Err(IoError::ReadFailed("short disk header".into())); - } - Ok(()) - }) - .await - .map_err(|error| IoError::ReadFailed(format!("DiskIO disk {id:?}: {error}"))) - }); - } - while let Some(read) = reads.join_next().await { - read.map_err(|error| IoError::ReadFailed(error.to_string()))??; - } - } + }, + ) + .await?; total.mark_success(); Ok(format!( - "ChunkDB read and 4096-byte DiskIO reads succeeded on {} online disks", - online.len() + "ChunkDB read and 4096-byte DiskIO reads succeeded on {online} online disks" )) } @@ -87,6 +57,99 @@ impl ChunkIoClient { pub fn readiness_metrics(&self) -> ReadinessMetricsSnapshot { self.readiness.snapshot() } + + #[cfg(feature = "test-util")] + /// Exercise production dependency scheduling with controlled authoritative observations. + pub async fn observe_readiness_dependencies_for_tests( + deadline: Instant, + metrics: &Arc, + writer: Arc, + hardware: impl Future>>, + chunkdb: impl Future>, + ) -> Result { + observe_dependencies(deadline, metrics, writer, hardware, chunkdb).await + } +} + +async fn observe_dependencies( + deadline: Instant, + metrics: &Arc, + writer: Arc, + hardware: impl Future>>, + chunkdb: impl Future>, +) -> Result { + let (disks, chunks) = tokio::join!( + async { + let disks = observe("hardware", deadline, &metrics.hardware, hardware).await?; + // Disk enumeration depends on hardware, not on the independent ChunkDB read. + observe_online_disks(disks, writer, deadline, metrics).await + }, + observe("ChunkDB", deadline, &metrics.chunkdb, chunkdb), + ); + chunks?; + disks +} + +async fn observe_online_disks( + disks: Vec, + writer: Arc, + deadline: Instant, + metrics: &Arc, +) -> Result { + let online: Vec<_> = disks + .iter() + .filter(|disk| disk.value.status == HwStatus::Up as i32) + .collect(); + if online.is_empty() { + return Err(IoError::Topology("no online disks are available".into())); + } + // Keep background observation work bounded independently of cluster size. + for batch in online.chunks(8) { + let mut reads = tokio::task::JoinSet::new(); + for disk in batch { + let writer = writer.clone(); + let metrics = metrics.clone(); + let id = disk.disk_id; + reads.spawn(async move { observe_disk(writer.as_ref(), id, deadline, &metrics).await }); + } + while let Some(read) = reads.join_next().await { + read.map_err(|error| IoError::ReadFailed(error.to_string()))??; + } + } + Ok(online.len()) +} + +async fn observe_disk( + writer: &dyn DiskWriter, + id: DiskId, + deadline: Instant, + metrics: &crate::metrics::ReadinessMetrics, +) -> Result<()> { + let segment = Segment { + disk_id: Some(id), + unit_count: 1, + ..Default::default() + }; + observe("DiskIO", deadline, &metrics.diskio, async { + let bytes = writer.read_priority(&segment, 4096, 0, 4096).await?; + if bytes.len() != 4096 { + return Err(IoError::ReadFailed("short disk header".into())); + } + Ok(()) + }) + .await + .map_err(|error| IoError::ReadFailed(format!("DiskIO disk {id:?}: {error}"))) +} + +#[cfg(feature = "test-util")] +/// Exercise the production physical probe with an injected routed transport. +pub async fn observe_disk_readiness_for_tests( + writer: &dyn DiskWriter, + id: DiskId, + deadline: Instant, + metrics: &crate::metrics::ReadinessMetrics, +) -> Result<()> { + observe_disk(writer, id, deadline, metrics).await } async fn observe( diff --git a/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs b/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs index 174d93b10..f4479a00a 100644 --- a/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs +++ b/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs @@ -193,6 +193,17 @@ pub trait DiskWriter: Send + Sync { )) } + /// Read a bounded control range through the independently provisioned priority lane. + async fn read_priority( + &self, + seg: &Segment, + unit_bytes: u64, + segment_offset: u64, + length: u32, + ) -> Result { + self.read(seg, unit_bytes, segment_offset, length).await + } + /// Read while preserving backing metadata when the transport provides it. async fn read_payload( &self, diff --git a/lib/crowdb-chunk-client/src/disk_io/routing.rs b/lib/crowdb-chunk-client/src/disk_io/routing.rs index a3aa87255..17ba4af27 100644 --- a/lib/crowdb-chunk-client/src/disk_io/routing.rs +++ b/lib/crowdb-chunk-client/src/disk_io/routing.rs @@ -9,12 +9,15 @@ use async_trait::async_trait; use bytes::Bytes; use crowdb_diskio_client::{ DiskId, DiskioClient, DiskioClientConfig, DiskioError, Durability, NativeDiskIoRoutes, SegmentTarget, + TrafficLane, }; use crowdb_kv_client::{HardwareClient, ServiceRegistryClient}; use crowdb_protocol::diskdb::rpc::Segment; use crate::{DiskWriter, IoError, Result}; +mod reads; + /// Disk writer backed by the complete semantic DiskIO client. pub struct RoutedDiskWriter { client: Arc, @@ -370,24 +373,20 @@ impl DiskWriter for RoutedDiskWriter { segment_offset: u64, length: u32, ) -> Result { - let target = Self::target(seg, unit_bytes).map_err(|error| IoError::ReadFailed(error.to_string()))?; - let result = self - .client - .read_payload(target, segment_offset, length, self.client.normal_options()) - .await; - match result { - Err(DiskioError::TransportUnavailable(_) | DiskioError::TopologyUnavailable(_)) => { - // A restarted DiskIO can advertise a new endpoint while the - // retained generation still points at its old connection. - // Reads are safe to retry once after an authoritative refresh. - self.client.refresh().await.map_err(map_read_error)?; - self.client - .read_payload(target, segment_offset, length, self.client.normal_options()) - .await - .map_err(map_read_error) - } - result => result.map_err(map_read_error), - } + self.read_payload_in_lane(seg, unit_bytes, segment_offset, length, TrafficLane::Normal) + .await + } + + async fn read_priority( + &self, + seg: &Segment, + unit_bytes: u64, + segment_offset: u64, + length: u32, + ) -> Result { + self.read_payload_in_lane(seg, unit_bytes, segment_offset, length, TrafficLane::Priority) + .await + .map(crowdb_rpc_ffi::OwnedPayload::into_bytes) } } diff --git a/lib/crowdb-chunk-client/src/disk_io/routing/reads.rs b/lib/crowdb-chunk-client/src/disk_io/routing/reads.rs new file mode 100644 index 000000000..e7549ef35 --- /dev/null +++ b/lib/crowdb-chunk-client/src/disk_io/routing/reads.rs @@ -0,0 +1,44 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Preserve semantic routing and refresh behavior for both transport lanes. + +use crowdb_protocol::diskdb::rpc::Segment; +use crowdb_rpc_ffi::OwnedPayload; + +use super::{map_read_error, DiskioError, IoError, Result, RoutedDiskWriter, TrafficLane}; + +impl RoutedDiskWriter { + pub(super) async fn read_payload_in_lane( + &self, + seg: &Segment, + unit_bytes: u64, + segment_offset: u64, + length: u32, + lane: TrafficLane, + ) -> Result { + let target = Self::target(seg, unit_bytes).map_err(|error| IoError::ReadFailed(error.to_string()))?; + let options = || { + let mut options = self.client.normal_options(); + options.lane = lane; + options + }; + let result = self + .client + .read_payload(target, segment_offset, length, options()) + .await; + match result { + Err(DiskioError::TransportUnavailable(_) | DiskioError::TopologyUnavailable(_)) => { + // A restarted DiskIO can advertise a new endpoint while the + // retained generation still points at its old connection. + // Reads are safe to retry once after an authoritative refresh. + self.client.refresh().await.map_err(map_read_error)?; + self.client + .read_payload(target, segment_offset, length, options()) + .await + .map_err(map_read_error) + } + result => result.map_err(map_read_error), + } + } +} diff --git a/lib/crowdb-chunk-client/src/writer/small_pipeline.rs b/lib/crowdb-chunk-client/src/writer/small_pipeline.rs index b0fb92b64..24d91c855 100644 --- a/lib/crowdb-chunk-client/src/writer/small_pipeline.rs +++ b/lib/crowdb-chunk-client/src/writer/small_pipeline.rs @@ -76,6 +76,7 @@ impl ManagedPipeline { } mod batch; +mod liveness; mod prefetch; mod worker; @@ -659,25 +660,6 @@ impl OwnedChunk { self.policy.chunk_capacity.saturating_sub(self.cursor) } - async fn renew_liveness(&mut self) -> Result<()> { - self.flush_pending_advance().await?; - let chunk_id = self - .chunk - .id - .ok_or_else(|| IoError::AllocationFailed("shared chunk missing id".into()))?; - self.chunk = advance_chunk( - Arc::clone(&self.allocator), - chunk_id, - self.writer_epoch, - self.chunk.modify_ts, - self.cursor, - None, - u64::try_from(self.policy.writer_lease.as_millis()).unwrap_or(u64::MAX), - ) - .await?; - Ok(()) - } - fn current_strip(&self) -> Result<&crowdb_protocol::chunkdb::rpc::ChunkStrip> { self.chunk .strips diff --git a/lib/crowdb-chunk-client/src/writer/small_pipeline/liveness.rs b/lib/crowdb-chunk-client/src/writer/small_pipeline/liveness.rs new file mode 100644 index 000000000..f1125f1e1 --- /dev/null +++ b/lib/crowdb-chunk-client/src/writer/small_pipeline/liveness.rs @@ -0,0 +1,62 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{advance_chunk, Arc, ChunkState, IoError, OwnedChunk, Result}; + +impl OwnedChunk { + pub(super) async fn renew_liveness(&mut self) -> Result<()> { + let staged = self + .staged_reservation + .map(|(_, _, sequence)| { + self.chunk + .strips + .iter() + .find(|strip| strip.strip_sequence == sequence) + .cloned() + .ok_or_else(|| IoError::MetadataConflict("staged reservation strip is missing".into())) + }) + .transpose()?; + self.flush_pending_advance().await?; + let chunk_id = self + .chunk + .id + .ok_or_else(|| IoError::AllocationFailed("shared chunk missing id".into()))?; + let mut refreshed = advance_chunk( + Arc::clone(&self.allocator), + chunk_id, + self.writer_epoch, + self.chunk.modify_ts, + self.cursor, + None, + u64::try_from(self.policy.writer_lease.as_millis()).unwrap_or(u64::MAX), + ) + .await?; + if refreshed.id != Some(chunk_id) + || refreshed.writer_epoch != self.writer_epoch + || refreshed.state != ChunkState::Active as i32 + { + return Err(IoError::MetadataConflict( + "shared chunk ownership changed during lease renewal".into(), + )); + } + if let Some(staged) = staged { + // Unconsumed reservations are absent from the readable layout. + // Retain this local write view without advancing published capacity. + match refreshed + .strips + .iter() + .find(|strip| strip.strip_sequence == staged.strip_sequence) + { + Some(current) if current != &staged => { + return Err(IoError::MetadataConflict( + "staged reservation changed during lease renewal".into(), + )); + } + Some(_) => {} + None => refreshed.strips.push(staged), + } + } + self.chunk = refreshed; + Ok(()) + } +} diff --git a/lib/crowdb-chunk-client/tests/chunk_reader_test.rs b/lib/crowdb-chunk-client/tests/chunk_reader_test.rs index d8c5f2e0d..63f5ebe68 100644 --- a/lib/crowdb-chunk-client/tests/chunk_reader_test.rs +++ b/lib/crowdb-chunk-client/tests/chunk_reader_test.rs @@ -34,6 +34,12 @@ mod frame_revision; #[path = "chunk_reader_test/layout_cache.rs"] mod layout_cache; +#[path = "chunk_reader_test/mirror_locality.rs"] +mod mirror_locality; + +#[path = "common/mirror_locality.rs"] +mod mirror_locality_fixture; + struct MemoryDiskIo { shards: Vec<(DiskId, Bytes)>, failed: Vec, diff --git a/lib/crowdb-chunk-client/tests/chunk_reader_test/mirror_locality.rs b/lib/crowdb-chunk-client/tests/chunk_reader_test/mirror_locality.rs new file mode 100644 index 000000000..5192344b5 --- /dev/null +++ b/lib/crowdb-chunk-client/tests/chunk_reader_test/mirror_locality.rs @@ -0,0 +1,36 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; +use crate::mirror_locality_fixture::{fixture, read}; + +#[tokio::test] +async fn mirror_reads_prefer_the_requested_node_without_changing_metadata() { + let (reader, disks, allocator, frame) = fixture(Some(3), None, None); + read(&reader, &frame).await; + assert_eq!(*disks.served.lock().unwrap(), vec![3]); + assert_eq!( + allocator.current.lock().unwrap().strips[0].strip, + allocator.first.strips[0].strip + ); +} + +#[tokio::test] +async fn unknown_node_or_incomplete_hints_preserve_every_supplied_replica() { + for (preferred, missing) in [(None, None), (Some(99), None), (Some(3), Some(1))] { + let (reader, disks, _, frame) = fixture(preferred, missing, None); + read(&reader, &frame).await; + assert_eq!(*disks.served.lock().unwrap(), vec![1]); + } +} + +#[tokio::test] +async fn corrupt_preferred_mirror_falls_back_within_its_rack_after_reporting() { + let (reader, disks, allocator, frame) = fixture(Some(3), None, Some(3)); + read(&reader, &frame).await; + assert_eq!(*disks.served.lock().unwrap(), vec![3, 2]); + assert_eq!( + allocator.current.lock().unwrap().strips[0].unavailable_segments, + vec![segment(3)] + ); +} diff --git a/lib/crowdb-chunk-client/tests/common/mirror_locality.rs b/lib/crowdb-chunk-client/tests/common/mirror_locality.rs new file mode 100644 index 000000000..c34deb33f --- /dev/null +++ b/lib/crowdb-chunk-client/tests/common/mirror_locality.rs @@ -0,0 +1,124 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; +use crowdb_diskio_client::{DiskTopologyView, TestDiskRoute}; + +pub(super) struct TestLocalityDiskIo { + bytes: Bytes, + topology: Option, + pub(super) served: Mutex>, + corrupt: Option, +} + +#[async_trait] +impl DiskWriter for TestLocalityDiskIo { + fn disk_topology(&self) -> Option { + self.topology.clone() + } + + async fn write(&self, _: &Segment, _: u64, _: Bytes) -> Result<()> { + unreachable!("reader test does not write") + } + + async fn write_at_byte_offset(&self, _: &Segment, _: u64, _: u64, _: Bytes) -> Result<()> { + unreachable!("reader test does not write") + } + + async fn read(&self, segment: &Segment, _: u64, offset: u64, length: u32) -> Result { + let id = segment.disk_id.unwrap().low; + self.served.lock().unwrap().push(id); + let offset = usize::try_from(offset).unwrap(); + let length = usize::try_from(length).unwrap(); + let bytes = self.bytes.slice(offset..offset + length); + if self.corrupt == Some(id) { + let mut bytes = bytes.to_vec(); + bytes[crowdb_protocol::frame::FRAME_V2_HEADER_BYTES] ^= 1; + Ok(Bytes::from(bytes)) + } else { + Ok(bytes) + } + } +} + +pub(super) fn fixture( + preferred: Option, + missing: Option, + corrupt: Option, +) -> ( + ChunkReader, + Arc, + Arc, + Bytes, +) { + let id = ChunkId { high: 83, low: 89 }; + let frame = Bytes::from(encode_frame(FrameMagic::RepoSmallV2, id, b"exact payload", 1).unwrap()); + // Metadata deliberately starts in another rack, followed by a sibling + // node in the preferred rack and finally the preferred node itself. + let replicas = vec![segment(1), segment(2), segment(3)]; + let chunk = Chunk { + id: Some(id), + state: ChunkState::Active as i32, + modify_ts: 1, + capacity: 1, + acknowledged_cursor: frame.len() as u64, + strips: vec![ChunkStrip { + capacity: 1, + unit_kb: 1, + strip: Some(Strip::MirrorStrip(MirrorStrip { segments: replicas })), + ..ChunkStrip::default() + }], + ..Chunk::default() + }; + let allocator = Arc::new(SequenceAllocator { + queries: AtomicUsize::new(0), + first: chunk.clone(), + current: Mutex::new(chunk), + first_layout_validity_ms: 1_000, + full_reply: None, + full_calls: AtomicUsize::new(0), + }); + let disks = Arc::new(TestLocalityDiskIo { + bytes: frame.clone(), + topology: Some(DiskTopologyView::from_routes_for_tests( + (1..=3) + .filter(|id| Some(*id) != missing) + .map(|id| TestDiskRoute { + disk_id: crowdb_diskio_client::DiskId::new(0, id), + rack_id: if id == 1 { 2 } else { 1 }, + node_id: id, + disk_group_id: id, + instance_id: id, + endpoint: format!("127.0.0.1:{}", 12000 + id), + }) + .collect(), + )), + served: Mutex::new(Vec::new()), + corrupt, + }); + let reader = ChunkReader::new( + allocator.clone(), + disks.clone(), + ChunkReadPolicy { + preferred_node_id: preferred, + ..ChunkReadPolicy::default() + }, + ) + .unwrap(); + (reader, disks, allocator, frame) +} + +pub(super) async fn read(reader: &ChunkReader, frame: &Bytes) { + assert_eq!( + reader + .read_verified_frame( + ChunkId { high: 83, low: 89 }, + 0, + frame.len() as u64, + FrameMagic::RepoSmallV2, + ) + .await + .unwrap(), + *frame + ); +} diff --git a/lib/crowdb-chunk-client/tests/common/mirrored_frames.rs b/lib/crowdb-chunk-client/tests/common/mirrored_frames.rs index 2af4fa8b9..e11906deb 100644 --- a/lib/crowdb-chunk-client/tests/common/mirrored_frames.rs +++ b/lib/crowdb-chunk-client/tests/common/mirrored_frames.rs @@ -64,6 +64,8 @@ impl TestMirroredFrames { calls: AtomicUsize::new(0), short: AtomicBool::new(false), delay_ms: AtomicU64::new(0), + topology: None, + slow_disk: None, }); Self { chunk, @@ -79,10 +81,15 @@ pub struct TestFrameDisks { pub calls: AtomicUsize, pub short: AtomicBool, pub delay_ms: AtomicU64, + pub topology: Option, + pub slow_disk: Option<(u64, std::time::Duration)>, } #[async_trait] impl DiskWriter for TestFrameDisks { + fn disk_topology(&self) -> Option { + self.topology.clone() + } async fn write(&self, _: &Segment, _: u64, _: Bytes) -> crowdb_chunk_client::Result<()> { Err(IoError::WriteFailed("read-only observation fixture".into())) } @@ -104,6 +111,11 @@ impl DiskWriter for TestFrameDisks { ) -> crowdb_chunk_client::Result { self.calls.fetch_add(1, Ordering::Relaxed); let delay = self.delay_ms.load(Ordering::Relaxed); + if let Some((disk, duration)) = self.slow_disk { + if segment.disk_id.is_some_and(|id| id.high == disk) { + tokio::time::sleep(duration).await; + } + } if delay > 0 { tokio::time::sleep(std::time::Duration::from_millis(delay)).await; } diff --git a/lib/crowdb-chunk-client/tests/common/readiness_peer.rs b/lib/crowdb-chunk-client/tests/common/readiness_peer.rs new file mode 100644 index 000000000..62c7fe7d4 --- /dev/null +++ b/lib/crowdb-chunk-client/tests/common/readiness_peer.rs @@ -0,0 +1,172 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_diskio_client::{DiskId, DiskioClient, DiskioClientConfig, TestDiskRoute}; +use crowdb_protocol::diskio_fb::{ + FBDiskIoRetCode, FBDiskReadRequest, FBDiskReadResponse, FBDiskReadResponseArgs, +}; +use flatbuffers::FlatBufferBuilder; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::{TcpListener, TcpStream}, + sync::Notify, + task::{JoinHandle, JoinSet}, +}; + +#[derive(Clone, Copy)] +pub enum TestProbeReply { + Ready, + Rejected, + Held, +} + +#[derive(Default)] +pub struct TestReadState { + pub bulk_started: Notify, + pub release_bulk: Notify, + pub release_probe: Notify, + pub headers: AtomicUsize, +} + +pub struct TestReadinessPeer { + pub state: Arc, + address: String, + worker: JoinHandle<()>, +} + +impl TestReadinessPeer { + pub async fn start(reply: TestProbeReply) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap().to_string(); + let state = Arc::new(TestReadState::default()); + let worker = tokio::spawn({ + let state = state.clone(); + async move { + let mut connections = JoinSet::new(); + loop { + tokio::select! { + value=listener.accept()=>{let(stream,_)=value.unwrap();connections.spawn(serve(stream,state.clone(),reply));}, + value=connections.join_next(),if !connections.is_empty()=>{value.unwrap().unwrap();} + } + } + } + }); + Self { + state, + address, + worker, + } + } + + pub fn client(&self) -> Arc { + Arc::new( + DiskioClient::connect_for_tests( + vec![TestDiskRoute { + disk_id: DiskId::new(1, 2), + rack_id: 1, + node_id: 1, + disk_group_id: 1, + instance_id: 1, + endpoint: self.address.clone(), + }], + DiskioClientConfig { + normal_connections_per_endpoint: 1, + priority_connections_per_endpoint: 1, + retry_attempts: 1, + ..Default::default() + }, + ) + .unwrap(), + ) + } +} + +impl Drop for TestReadinessPeer { + fn drop(&mut self) { + self.worker.abort(); + } +} + +async fn serve(mut stream: TcpStream, state: Arc, reply: TestProbeReply) { + loop { + let mut header = [0; 14]; + if stream.read_exact(&mut header).await.is_err() { + return; + } + assert_eq!(u16::from_le_bytes(header[..2].try_into().unwrap()), 0xca70); + let kind = u16::from_le_bytes(header[2..4].try_into().unwrap()); + let length = u32::from_le_bytes(header[4..8].try_into().unwrap()) as usize; + assert!(length <= 4096); + assert_eq!(&header[8..12], &[0; 4]); + let mut control = vec![0; length]; + stream.read_exact(&mut control).await.unwrap(); + if kind == 0xfffe { + assert_eq!(control.len(), 160); + let mut capabilities = vec![0; 160]; + capabilities[..4].copy_from_slice(&[0x44, 0x52, 4, 0]); + capabilities[4..12].copy_from_slice(&1_u64.to_le_bytes()); + header[13] = 0; + if stream.write_all(&header).await.is_err() { + return; + } + if stream.write_all(&capabilities).await.is_err() { + return; + } + continue; + } + let request = flatbuffers::root::(&control).unwrap(); + let id = request.id(); + let size = request.size() as usize; + assert!(matches!(size, 4096 | 65536)); + let bulk = size == 65536; + if !bulk { + state.headers.fetch_add(1, Ordering::Release); + if matches!(reply, TestProbeReply::Held) { + state.release_probe.notified().await; + } + } + let code = if !bulk && matches!(reply, TestProbeReply::Rejected) { + FBDiskIoRetCode::DiskNotExist + } else { + FBDiskIoRetCode::Success + }; + let mut builder = FlatBufferBuilder::new(); + let value = FBDiskReadResponse::create( + &mut builder, + &FBDiskReadResponseArgs { + id, + ret_code: code, + ..Default::default() + }, + ); + builder.finish(value, None); + let control = builder.finished_data(); + let bytes = if code == FBDiskIoRetCode::Success { size } else { 0 }; + let mut response = Vec::with_capacity(14 + control.len()); + response.extend_from_slice(&0xca70_u16.to_le_bytes()); + response.extend_from_slice(&(kind + 1).to_le_bytes()); + response.extend_from_slice(&u32::try_from(control.len()).unwrap().to_le_bytes()); + response.extend_from_slice(&u32::try_from(bytes).unwrap().to_le_bytes()); + response.extend_from_slice(&[14, 0]); + response.extend_from_slice(control); + if stream.write_all(&response).await.is_err() { + return; + } + if bulk { + if stream.write_all(&[7]).await.is_err() { + return; + } + state.bulk_started.notify_one(); + state.release_bulk.notified().await; + if stream.write_all(&vec![7; bytes - 1]).await.is_err() { + return; + } + } else if bytes != 0 && stream.write_all(&vec![9; bytes]).await.is_err() { + return; + } + } +} diff --git a/lib/crowdb-chunk-client/tests/common/readiness_schedule.rs b/lib/crowdb-chunk-client/tests/common/readiness_schedule.rs new file mode 100644 index 000000000..c1822fc8d --- /dev/null +++ b/lib/crowdb-chunk-client/tests/common/readiness_schedule.rs @@ -0,0 +1,106 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use async_trait::async_trait; +use bytes::Bytes; +use crowdb_chunk_client::{DiskWriter, IoError, Result}; +use crowdb_kv_client::DiskRecord; +use crowdb_protocol::{ + common::{DiskId, HwStatus}, + diskdb::rpc::{DiskValue, Segment}, +}; +use std::{ + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; + +pub enum TestHeaderReply { + Ready, + Rejected, + Short, + Held, +} + +pub struct TestReadinessWriter { + pub delay: Duration, + pub reply: TestHeaderReply, + pub started: AtomicUsize, + pub active: Arc, + pub maximum: AtomicUsize, + pub read_started: tokio::sync::Notify, +} + +impl TestReadinessWriter { + pub fn new(delay: Duration, reply: TestHeaderReply) -> Arc { + Arc::new(Self { + delay, + reply, + started: AtomicUsize::new(0), + active: Arc::default(), + maximum: AtomicUsize::new(0), + read_started: tokio::sync::Notify::new(), + }) + } +} + +struct TestActiveRead(Arc); +impl Drop for TestActiveRead { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::SeqCst); + } +} + +#[async_trait] +impl DiskWriter for TestReadinessWriter { + async fn write(&self, _: &Segment, _: u64, _: Bytes) -> Result<()> { + panic!("readiness must not write") + } + + async fn write_at_byte_offset(&self, _: &Segment, _: u64, _: u64, _: Bytes) -> Result<()> { + panic!("readiness must not write") + } + + async fn read_priority( + &self, + segment: &Segment, + unit_bytes: u64, + offset: u64, + length: u32, + ) -> Result { + assert!(segment.disk_id.is_some()); + assert_eq!((unit_bytes, offset, length), (4096, 0, 4096)); + self.started.fetch_add(1, Ordering::SeqCst); + let active = self.active.fetch_add(1, Ordering::SeqCst) + 1; + self.maximum.fetch_max(active, Ordering::SeqCst); + let _active = TestActiveRead(self.active.clone()); + self.read_started.notify_one(); + tokio::time::sleep(self.delay).await; + match self.reply { + TestHeaderReply::Ready => Ok(Bytes::from(vec![7; 4096])), + TestHeaderReply::Rejected => Err(IoError::ReadFailed("injected physical rejection".into())), + TestHeaderReply::Short => Ok(Bytes::from_static(b"short")), + TestHeaderReply::Held => std::future::pending().await, + } + } +} + +pub fn test_disks(count: usize) -> Vec { + (0..count) + .map(|n| DiskRecord { + rack_id: 1, + node_id: 1, + disk_group_id: n as u64 + 1, + disk_id: DiskId { + high: 1, + low: n as u64 + 1, + }, + value: DiskValue { + status: HwStatus::Up as i32, + ..Default::default() + }, + }) + .collect() +} diff --git a/lib/crowdb-chunk-client/tests/frame_observation_test/mirrored.rs b/lib/crowdb-chunk-client/tests/frame_observation_test/mirrored.rs index ae512c44f..b9e454e4b 100644 --- a/lib/crowdb-chunk-client/tests/frame_observation_test/mirrored.rs +++ b/lib/crowdb-chunk-client/tests/frame_observation_test/mirrored.rs @@ -8,6 +8,66 @@ use std::sync::atomic::Ordering; use super::*; use crate::mirrored_fixture::TestMirroredFrames as Layout; +#[tokio::test(start_paused = true)] +async fn alternating_mirror_order_cannot_force_every_observation_across_a_slow_rack() { + use crowdb_diskio_client::{DiskId, DiskTopologyView, TestDiskRoute}; + + let mut layout = Layout::new(); + for (index, strip) in layout.chunk.strips.iter_mut().enumerate() { + if index % 2 == 1 { + let Some(Strip::MirrorStrip(mirrors)) = &mut strip.strip else { + panic!("mirror layout"); + }; + mirrors.segments.reverse(); + } + } + let topology = DiskTopologyView::from_routes_for_tests( + (1..=2) + .map(|disk| TestDiskRoute { + disk_id: DiskId::new(disk, 1), + rack_id: 3 - disk, + node_id: disk, + disk_group_id: disk, + instance_id: disk, + endpoint: format!("127.0.0.1:{}", 9 + disk), + }) + .collect(), + ); + let disks = Arc::get_mut(&mut layout.disks).unwrap(); + disks.topology = Some(topology); + disks.slow_disk = Some((1, Duration::from_millis(60))); + disks.delay_ms.store(1, Ordering::Relaxed); + let original = Arc::new(layout.chunk); + let deadline = Instant::now() + Duration::from_millis(100); + let readers = + MirroredFrameReaders::new(original.clone(), Layout::NAME, layout.disks.clone(), deadline).unwrap(); + let selected = select_frame_prefix( + readers.readers(), + readers.scan_config(0, 0, 0, 64 * 1024 * 1024).unwrap(), + 65536, + deadline, + ) + .await + .unwrap(); + assert_eq!(selected.healthy_replicas(), 1); + assert_eq!(selected.observation().summary().physical_end, layout.body_length); + let mut sink = TestExtentSink::default(); + replay_frame_prefix( + readers.readers()[selected.selected_index()].as_ref(), + selected.observation(), + deadline, + &mut sink, + ) + .await + .unwrap(); + assert_eq!(sink.records.len(), 1); + assert!(Instant::now() < deadline); + let Some(Strip::MirrorStrip(mirrors)) = &original.strips[1].strip else { + panic!("mirror layout"); + }; + assert_eq!(mirrors.segments[0].disk_id.unwrap().high, 2); +} + #[tokio::test] async fn pinned_layout_reads_grouped_data_beyond_old_cursor_across_frames_and_strips() { let layout = Layout::new(); @@ -41,6 +101,35 @@ async fn pinned_layout_reads_grouped_data_beyond_old_cursor_across_frames_and_st assert_eq!(sink.records[0].logical_length, 100_000); } +#[tokio::test(start_paused = true)] +async fn incomplete_topology_hints_preserve_even_the_unmapped_replica() { + use crowdb_diskio_client::{DiskId, DiskTopologyView, TestDiskRoute}; + + let mut layout = Layout::new(); + let disks = Arc::get_mut(&mut layout.disks).unwrap(); + disks.topology = Some(DiskTopologyView::from_routes_for_tests(vec![TestDiskRoute { + disk_id: DiskId::new(2, 1), + rack_id: 1, + node_id: 2, + disk_group_id: 2, + instance_id: 2, + endpoint: "127.0.0.1:11".into(), + }])); + disks.slow_disk = Some((1, Duration::from_millis(40))); + let deadline = Instant::now() + Duration::from_millis(20); + let readers = MirroredFrameReaders::new( + Arc::new(layout.chunk), + Layout::NAME, + layout.disks.clone(), + deadline, + ) + .unwrap(); + assert_eq!(readers.readers().len(), 2); + assert!(readers.readers()[0].read_window(0, 512).await.is_err()); + assert_eq!(Instant::now(), deadline); + assert_eq!(layout.disks.calls.load(Ordering::Relaxed), 1); +} + #[tokio::test] async fn unavailable_replica_is_excluded_without_reading_or_mutating_its_layout() { let mut layout = Layout::new(); diff --git a/lib/crowdb-chunk-client/tests/readiness_priority_test.rs b/lib/crowdb-chunk-client/tests/readiness_priority_test.rs new file mode 100644 index 000000000..9114b23de --- /dev/null +++ b/lib/crowdb-chunk-client/tests/readiness_priority_test.rs @@ -0,0 +1,103 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/readiness_peer.rs"] +mod peer; +#[path = "common/read_payload.rs"] +mod storage; + +use crowdb_chunk_client::{ + client::observe_disk_readiness_for_tests, metrics::ReadinessMetrics, ChunkClientMetrics, ChunkIoClient, + RoutedDiskWriter, +}; +use crowdb_common::metrics::{MetricPoint, MetricsRegistry}; +use crowdb_diskio_client::{DiskId, OperationOptions, SegmentTarget}; +use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkType}; +use crowdb_protocol::common::DiskId as ProtoDiskId; +use crowdb_rpc_ffi::OwnedPayload; +use peer::{TestProbeReply, TestReadinessPeer}; +use std::{ + sync::{atomic::Ordering, Arc}, + time::Duration, +}; +use tokio::time::Instant; + +async fn exercise(reply: TestProbeReply) -> (crowdb_chunk_client::Result<()>, usize, ReadinessMetrics) { + let peer = TestReadinessPeer::start(reply).await; + let client = peer.client(); + let target = SegmentTarget::new(DiskId::new(1, 2), 0, 0, 16, 4096).unwrap(); + let bulk = tokio::spawn({ + let client = client.clone(); + async move { + client + .read(target, 0, 65536, OperationOptions::within(Duration::from_secs(5))) + .await + } + }); + tokio::time::timeout(Duration::from_secs(2), peer.state.bulk_started.notified()) + .await + .unwrap(); + let mut registry = MetricsRegistry::new(); + let aggregate = Arc::new(ChunkClientMetrics::register(&mut registry)); + let io = ChunkIoClient::from_parts( + Arc::new(storage::TestPayloadStorage { + chunk: Chunk::default(), + data: OwnedPayload::from(Vec::new()), + }), + Arc::new(RoutedDiskWriter::from_client(client.clone())), + ChunkType::S3, + ) + .with_metrics(&aggregate); + let (_, writer) = io.storage_parts(); + let metrics = ReadinessMetrics::default(); + let result = observe_disk_readiness_for_tests( + writer.as_ref(), + ProtoDiskId { high: 1, low: 2 }, + Instant::now() + Duration::from_millis(750), + &metrics, + ) + .await; + assert!(matches!( + registry.snapshot_named("chunkio.diskio.read.e2e.lh", 1.0), + Some(MetricPoint::Histogram { total: 1, .. }) + )); + assert!( + !bulk.is_finished(), + "the bulk response must remain held while readiness completes" + ); + let headers = peer.state.headers.load(Ordering::Acquire); + peer.state.release_probe.notify_one(); + peer.state.release_bulk.notify_one(); + let bytes = bulk.await.unwrap().unwrap(); + assert_eq!(bytes.len(), 65536); + assert!(bytes.iter().all(|b| *b == 7)); + client.shutdown(); + (result, headers, metrics) +} + +#[tokio::test] +async fn readiness_header_progresses_while_a_bulk_response_is_held() { + let (result, headers, metrics) = exercise(TestProbeReply::Ready).await; + assert!( + result.is_ok(), + "physical readiness must not wait behind a bulk TCP response: {result:?}" + ); + assert_eq!(headers, 1); + assert_eq!(metrics.snapshot().diskio.errors, 0); +} + +#[tokio::test] +async fn rejected_priority_header_still_fails_readiness() { + let (result, headers, metrics) = exercise(TestProbeReply::Rejected).await; + assert!(result.is_err()); + assert_eq!(headers, 1); + assert_eq!(metrics.snapshot().diskio.errors, 1); +} + +#[tokio::test] +async fn held_priority_header_still_owes_the_original_readiness_deadline() { + let (result, headers, metrics) = exercise(TestProbeReply::Held).await; + assert!(result.unwrap_err().to_string().contains("readiness deadline")); + assert_eq!(headers, 1); + assert_eq!(metrics.snapshot().diskio.errors, 1); +} diff --git a/lib/crowdb-chunk-client/tests/readiness_test.rs b/lib/crowdb-chunk-client/tests/readiness_test.rs index 9c6d96bba..efe59583f 100644 --- a/lib/crowdb-chunk-client/tests/readiness_test.rs +++ b/lib/crowdb-chunk-client/tests/readiness_test.rs @@ -1,12 +1,238 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use std::sync::Arc; +use std::sync::{atomic::Ordering, Arc}; use std::time::Duration; -use crowdb_chunk_client::{client::observe_readiness_phase_for_tests, metrics::ReadinessMetrics}; +use crowdb_chunk_client::{ + client::observe_readiness_phase_for_tests, metrics::ReadinessMetrics, ChunkIoClient, IoError, +}; use tokio::time::Instant; +#[path = "common/readiness_schedule.rs"] +mod schedule; + +#[tokio::test(start_paused = true)] +async fn disk_headers_start_after_hardware_without_waiting_for_chunkdb() { + let metrics = Arc::new(ReadinessMetrics::default()); + let writer = + schedule::TestReadinessWriter::new(Duration::from_millis(100), schedule::TestHeaderReply::Ready); + let start = Instant::now(); + let result = crowdb_chunk_client::ChunkIoClient::observe_readiness_dependencies_for_tests( + start + Duration::from_millis(750), + &metrics, + writer.clone(), + async { + tokio::time::sleep(Duration::from_millis(50)).await; + Ok(schedule::test_disks(1)) + }, + async { + tokio::time::sleep(Duration::from_millis(700)).await; + Ok(()) + }, + ) + .await; + assert!( + result.is_ok(), + "independent hardware/header and ChunkDB work must fit the original 750ms total: {result:?}" + ); + assert_eq!(result.unwrap(), 1); + assert_eq!(Instant::now() - start, Duration::from_millis(700)); + assert_eq!(writer.started.load(Ordering::SeqCst), 1); + assert_eq!(metrics.snapshot().diskio.errors, 0); +} + +#[tokio::test(start_paused = true)] +async fn held_chunkdb_still_fails_after_a_successful_physical_header() { + let metrics = Arc::new(ReadinessMetrics::default()); + let writer = + schedule::TestReadinessWriter::new(Duration::from_millis(100), schedule::TestHeaderReply::Ready); + let deadline = Instant::now() + Duration::from_millis(750); + let result = ChunkIoClient::observe_readiness_dependencies_for_tests( + deadline, + &metrics, + writer.clone(), + async { Ok(schedule::test_disks(1)) }, + std::future::pending(), + ) + .await; + assert!(result.unwrap_err().to_string().contains("ChunkDB read exceeded")); + assert_eq!(Instant::now(), deadline); + assert_eq!(writer.started.load(Ordering::SeqCst), 1); + assert_eq!(metrics.snapshot().chunkdb.errors, 1); + assert_eq!(metrics.snapshot().diskio.errors, 0); +} + +#[tokio::test(start_paused = true)] +async fn rejected_chunkdb_still_fails_after_a_successful_physical_header() { + let metrics = Arc::new(ReadinessMetrics::default()); + let writer = + schedule::TestReadinessWriter::new(Duration::from_millis(100), schedule::TestHeaderReply::Ready); + let result = ChunkIoClient::observe_readiness_dependencies_for_tests( + Instant::now() + Duration::from_millis(750), + &metrics, + writer.clone(), + async { Ok(schedule::test_disks(1)) }, + async { Err(IoError::ReadFailed("injected ChunkDB rejection".into())) }, + ) + .await; + assert!(result + .unwrap_err() + .to_string() + .contains("injected ChunkDB rejection")); + assert_eq!(writer.started.load(Ordering::SeqCst), 1); + assert_eq!(metrics.snapshot().chunkdb.errors, 1); + assert_eq!(metrics.snapshot().diskio.errors, 0); +} + +#[tokio::test(start_paused = true)] +async fn held_physical_header_still_owes_the_original_total_deadline() { + let metrics = Arc::new(ReadinessMetrics::default()); + let writer = schedule::TestReadinessWriter::new(Duration::ZERO, schedule::TestHeaderReply::Held); + let deadline = Instant::now() + Duration::from_millis(750); + let result = ChunkIoClient::observe_readiness_dependencies_for_tests( + deadline, + &metrics, + writer.clone(), + async { Ok(schedule::test_disks(1)) }, + async { Ok(()) }, + ) + .await; + assert!(result.unwrap_err().to_string().contains("DiskIO read exceeded")); + assert_eq!(Instant::now(), deadline); + assert_eq!(metrics.snapshot().diskio.errors, 1); + assert_eq!(writer.active.load(Ordering::SeqCst), 0); +} + +#[tokio::test(start_paused = true)] +async fn rejected_or_short_physical_headers_cannot_pass_with_ready_chunkdb() { + for (reply, message) in [ + (schedule::TestHeaderReply::Rejected, "injected physical rejection"), + (schedule::TestHeaderReply::Short, "short disk header"), + ] { + let metrics = Arc::new(ReadinessMetrics::default()); + let writer = schedule::TestReadinessWriter::new(Duration::ZERO, reply); + let result = ChunkIoClient::observe_readiness_dependencies_for_tests( + Instant::now() + Duration::from_millis(750), + &metrics, + writer.clone(), + async { Ok(schedule::test_disks(1)) }, + async { Ok(()) }, + ) + .await; + assert!(result.unwrap_err().to_string().contains(message)); + assert_eq!(metrics.snapshot().diskio.errors, 1); + assert_eq!(writer.started.load(Ordering::SeqCst), 1); + } +} + +#[tokio::test(start_paused = true)] +async fn failed_or_held_hardware_never_starts_a_physical_read() { + for held in [false, true] { + let metrics = Arc::new(ReadinessMetrics::default()); + let writer = schedule::TestReadinessWriter::new(Duration::ZERO, schedule::TestHeaderReply::Ready); + let result = ChunkIoClient::observe_readiness_dependencies_for_tests( + Instant::now() + Duration::from_millis(750), + &metrics, + writer.clone(), + async { + if held { + std::future::pending().await + } else { + Err(IoError::Topology("injected hardware rejection".into())) + } + }, + async { Ok(()) }, + ) + .await; + let message = result.unwrap_err().to_string(); + assert!(message.contains(if held { + "hardware read exceeded" + } else { + "injected hardware rejection" + })); + assert_eq!(metrics.snapshot().hardware.errors, 1); + assert_eq!(metrics.snapshot().diskio.observations, 0); + assert_eq!(writer.started.load(Ordering::SeqCst), 0); + } +} + +#[tokio::test(start_paused = true)] +async fn empty_or_offline_hardware_does_not_claim_physical_readiness() { + for count in [0, 1] { + let metrics = Arc::new(ReadinessMetrics::default()); + let writer = schedule::TestReadinessWriter::new(Duration::ZERO, schedule::TestHeaderReply::Ready); + let mut disks = schedule::test_disks(count); + for disk in &mut disks { + disk.value.status = crowdb_protocol::common::HwStatus::Offline as i32; + } + let result = ChunkIoClient::observe_readiness_dependencies_for_tests( + Instant::now() + Duration::from_millis(750), + &metrics, + writer.clone(), + async { Ok(disks) }, + async { Ok(()) }, + ) + .await; + assert!(result.unwrap_err().to_string().contains("no online disks")); + assert_eq!(writer.started.load(Ordering::SeqCst), 0); + assert_eq!(metrics.snapshot().diskio.observations, 0); + } +} + +#[tokio::test(start_paused = true)] +async fn physical_headers_keep_the_existing_eight_read_batch_bound() { + let metrics = Arc::new(ReadinessMetrics::default()); + let writer = + schedule::TestReadinessWriter::new(Duration::from_millis(100), schedule::TestHeaderReply::Ready); + let start = Instant::now(); + let result = ChunkIoClient::observe_readiness_dependencies_for_tests( + start + Duration::from_millis(750), + &metrics, + writer.clone(), + async { Ok(schedule::test_disks(17)) }, + async { Ok(()) }, + ) + .await; + assert_eq!(result.unwrap(), 17); + assert_eq!(writer.started.load(Ordering::SeqCst), 17); + assert_eq!(writer.maximum.load(Ordering::SeqCst), 8); + assert_eq!(Instant::now() - start, Duration::from_millis(300)); + assert_eq!(metrics.snapshot().diskio.observations, 17); +} + +#[tokio::test(start_paused = true)] +async fn cancelling_dependency_observation_aborts_physical_reads_and_both_phase_guards() { + let metrics = Arc::new(ReadinessMetrics::default()); + let writer = schedule::TestReadinessWriter::new(Duration::ZERO, schedule::TestHeaderReply::Held); + let worker = tokio::spawn({ + let metrics = metrics.clone(); + let writer = writer.clone(); + async move { + ChunkIoClient::observe_readiness_dependencies_for_tests( + Instant::now() + Duration::from_millis(750), + &metrics, + writer, + async { Ok(schedule::test_disks(1)) }, + std::future::pending(), + ) + .await + } + }); + writer.read_started.notified().await; + assert_eq!(metrics.snapshot().chunkdb.inflight, 1); + assert_eq!(metrics.snapshot().diskio.inflight, 1); + worker.abort(); + assert!(worker.await.unwrap_err().is_cancelled()); + tokio::task::yield_now().await; + assert_eq!(writer.active.load(Ordering::SeqCst), 0); + let snapshot = metrics.snapshot(); + assert_eq!(snapshot.chunkdb.inflight, 0); + assert_eq!(snapshot.chunkdb.errors, 1); + assert_eq!(snapshot.diskio.inflight, 0); + assert_eq!(snapshot.diskio.errors, 1); +} + #[tokio::test(start_paused = true)] async fn phase_deadlines_share_the_remaining_budget_and_identify_the_dependency() { let metrics = ReadinessMetrics::default(); diff --git a/lib/crowdb-chunk-client/tests/small_object_test.rs b/lib/crowdb-chunk-client/tests/small_object_test.rs index 96fa371ed..0896d45b9 100644 --- a/lib/crowdb-chunk-client/tests/small_object_test.rs +++ b/lib/crowdb-chunk-client/tests/small_object_test.rs @@ -33,6 +33,9 @@ mod durable; #[path = "common/small_native.rs"] mod native; +#[path = "small_object_test/staged_reservation.rs"] +mod staged_reservation; + fn frame_bytes(payload_bytes: usize) -> u64 { u64::try_from(payload_bytes).unwrap() + 35 } @@ -109,7 +112,7 @@ impl ChunkAllocator for MockAllocator { high: u64::try_from(req.chunk_type).unwrap() << 56, low, }); - let strips: Vec<_> = (0..req.strip_count.max(1)) + let strips: Vec<_> = (0..req.strip_count) .map(|sequence| make_strip(chunk_id, sequence, req.copy_count.max(1))) .collect(); let chunk = Chunk { @@ -126,7 +129,7 @@ impl ChunkAllocator for MockAllocator { acknowledged_cursor: 0, closed_strip_sequence: None, writer_lease_deadline_ms: req.writer_lease_ms, - next_strip_sequence: req.strip_count.max(1), + next_strip_sequence: req.strip_count, cleanup_intents: vec![], last_strip_replacement: None, owner_key: req.owner_key, diff --git a/lib/crowdb-chunk-client/tests/small_object_test/staged_reservation.rs b/lib/crowdb-chunk-client/tests/small_object_test/staged_reservation.rs new file mode 100644 index 000000000..15202c44c --- /dev/null +++ b/lib/crowdb-chunk-client/tests/small_object_test/staged_reservation.rs @@ -0,0 +1,117 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; + +#[tokio::test(start_paused = true)] +async fn lease_renewal_preserves_the_unpublished_first_strip() { + let mut configured = policy(); + configured.conversion_enabled = true; + configured.memory_budget = 64 * 1024 * 1024; + configured.chunk_capacity = 2 * 1024 * 1024; + configured.small_strip_prefetch_count = 2; + let renewal = configured.writer_lease / 2; + let (client, allocator, disk) = client(configured); + let mut writer = client.prepare_small_write(16).await.unwrap(); + for _ in 0..1000 { + if allocator.reserve_calls.load(Ordering::Relaxed) > 0 { + break; + } + tokio::task::yield_now().await; + } + assert_eq!(allocator.reserve_calls.load(Ordering::Relaxed), 1); + assert_eq!( + allocator + .state + .lock() + .unwrap() + .chunks + .values() + .next() + .unwrap() + .capacity, + 0 + ); + tokio::time::sleep(renewal + Duration::from_millis(1)).await; + for _ in 0..1000 { + if allocator.snapshot().2 > 0 { + break; + } + tokio::task::yield_now().await; + } + assert!( + allocator.snapshot().2 > 0, + "the empty writer must renew before submission" + ); + { + let state = allocator.state.lock().unwrap(); + let published = state.chunks.values().next().unwrap(); + assert_eq!(published.capacity, 0); + assert!( + published.strips.is_empty(), + "renewal must not publish an unwritten strip" + ); + } + writer + .on_data(Bytes::from_static(b"0123456789abcdef")) + .await + .unwrap(); + let locations = writer.finish_durable().await.unwrap(); + assert_eq!(locations.len(), 1); + assert_eq!(locations[0].offset, 0); + for (_, _, image) in disk.writes.lock().unwrap().iter() { + let frame = parse_frame(image, locations[0].chunk_id.unwrap()).unwrap(); + assert_eq!(frame.payload, b"0123456789abcdef"); + } + assert_eq!(disk.calls(), 3); + client.shutdown_small_writes().await.unwrap(); +} + +#[tokio::test(start_paused = true)] +async fn an_idle_prepared_replacement_still_writes_at_zero_after_renewal() { + let mut configured = policy(); + configured.conversion_enabled = true; + configured.memory_budget = 64 * 1024 * 1024; + configured.chunk_capacity = 2 * 1024 * 1024; + configured.small_strip_prefetch_count = 2; + let renewal = configured.writer_lease / 2; + let (client, allocator, _) = client(configured); + let payload = Bytes::from(vec![0x5a; MAX_SMALL]); + let mut writer = client.prepare_small_write(MAX_SMALL).await.unwrap(); + writer.on_data(payload.clone()).await.unwrap(); + let original = writer.finish_durable().await.unwrap()[0].chunk_id; + for _ in 0..1000 { + if allocator.snapshot().0 == 2 { + break; + } + tokio::task::yield_now().await; + } + assert_eq!( + allocator.snapshot().0, + 2, + "a replacement must be prepared before renewal" + ); + tokio::time::sleep(renewal + Duration::from_millis(1)).await; + { + let state = allocator.state.lock().unwrap(); + assert!(state.advances >= 2, "current and prepared chunks must both renew"); + let replacement = state.chunks.get(&(original.unwrap().high, 2)).unwrap(); + assert!(replacement.modify_ts >= 3, "the prepared replacement must renew"); + assert_eq!(replacement.capacity, 0); + assert!(replacement.strips.is_empty()); + } + for _ in 1..32 { + let mut writer = client.prepare_small_write(MAX_SMALL).await.unwrap(); + writer.on_data(payload.clone()).await.unwrap(); + assert_eq!(writer.finish_durable().await.unwrap()[0].chunk_id, original); + } + let mut writer = client.prepare_small_write(16).await.unwrap(); + writer + .on_data(Bytes::from_static(b"replacement-data")) + .await + .unwrap(); + let location = writer.finish_durable().await.unwrap().remove(0); + assert_ne!(location.chunk_id, original); + assert_eq!(location.offset, 0); + client.shutdown_small_writes().await.unwrap(); +} diff --git a/lib/crowdb-chunk-kv/src/partition.rs b/lib/crowdb-chunk-kv/src/partition.rs index a9b3af1b1..c5b9db0c0 100644 --- a/lib/crowdb-chunk-kv/src/partition.rs +++ b/lib/crowdb-chunk-kv/src/partition.rs @@ -12,7 +12,9 @@ mod frontier; mod ingress; mod journal; mod maintenance; +mod native; mod observation; +mod replay; mod scan; mod split; mod transfer; @@ -23,7 +25,9 @@ pub use frame::{decode_frame, encode_frame, DecodedFrame, FrameDecode, MAX_FRAME pub use frontier::JournalFrontier; pub use ingress::SplitIngress; pub use journal::{PartitionJournal, StreamPartitionJournal}; +use native::native_storage_parts; pub use observation::TreeObservation; +use replay::{replay_suffix, validate_replay_record, ReplayState}; pub use split::{PreparedSplit, PreparedSplitWriter, SplitHandoffStore, SplitWriterTarget}; pub use tree::{CrowdbPartitionTree, PartitionTree}; use worker_sender::WorkerSender; @@ -228,15 +232,6 @@ struct RecoverySeed { recovered: bool, } -struct ReplayState { - seed: RecoverySeed, - checkpoint_applied_seq: u64, - stream_name: StreamName, - retained_results: usize, - replayed: HashMap, - last_new_sequence: Option, -} - struct SplitTransition { plan: SplitPlan, artifact: Option, @@ -517,33 +512,13 @@ impl Partition { "tree identity and ownership epoch must be nonzero".into(), )); } - let replay_offset = if page_store.is_chunk_backed() && tree::chunk_root_generation(&page_store)? == 0 - { - 0 - } else { - page_store.wal_replay_offset().map_err(|error| match error { - crowdb_tree_ffi::CtError::Corruption => ChunkKvError::TreeCorruption(error.to_string()), - _ => ChunkKvError::TreeUnavailable(error.to_string()), - })? - }; - let stream_name = stream.stream_name(); - let stream_generation = stream.record_generation(); - let (tree, journal) = native_storage_parts(tree_id, &range, tree_config, page_store, stream)?; - let (tree_manifest, applied_seq) = tree.checkpoint_state()?; - let root_manifest_generation = tree.root_manifest_generation()?; + let (tree, journal, checkpoint) = + native::open_latest(tree_id, range.clone(), tree_config, page_store, stream).await?; Self::recover_assignment( partition_id, range, ownership_epoch, - Checkpoint { - tree_id, - tree_manifest, - root_manifest_generation, - applied_seq, - stream_name, - stream_generation, - replay_offset, - }, + checkpoint, config, tree, journal, @@ -572,7 +547,7 @@ impl Partition { )); } let (tree, journal) = - native_storage_parts(checkpoint.tree_id, &range, tree_config, page_store, stream)?; + native::open_storage(checkpoint.tree_id, range.clone(), tree_config, page_store, stream).await?; Self::recover_assignment( partition_id, range, @@ -751,7 +726,7 @@ impl Partition { }; let range = artifact.range.clone(); let (tree, journal) = - native_storage_parts(artifact.tree_id, &range, tree_config, page_store, stream)?; + native::open_storage(artifact.tree_id, range, tree_config, page_store, stream).await?; let parent_journal: Arc = Arc::new(StreamPartitionJournal::new( parent_stream, artifact.parent_stream_name, @@ -1729,24 +1704,6 @@ impl Partition { } } -fn native_storage_parts( - tree_id: u64, - range: &PartitionRange, - mut tree_config: crowdb_tree_ffi::Config, - page_store: Arc, - stream: ChunkStream, -) -> Result<(Arc, Arc)> { - tree_config.page_store = Some(page_store); - tree_config.key_range = crowdb_tree_ffi::KeyRange::Bounded { - start: range.start.clone(), - end: range.end.clone(), - }; - let tree: Arc = Arc::new(CrowdbPartitionTree::open(tree_id, &tree_config)?); - let stream_name = stream.stream_name(); - let journal: Arc = Arc::new(StreamPartitionJournal::new(stream, stream_name)?); - Ok((tree, journal)) -} - fn validate_split_artifact(plan: &SplitPlan, artifact: &SplitArtifact, cutover_seq: u64) -> Result<()> { if artifact.transition_id != plan.transition_id || artifact.parent_id != plan.parent_id @@ -1900,6 +1857,7 @@ async fn replay_child_overlay( let mut replay = ReplayState { seed, checkpoint_applied_seq: cutover_seq, + applied_bytes_since_drain: 0, stream_name: journal.stream_name(), retained_results, replayed: HashMap::new(), @@ -1935,185 +1893,6 @@ async fn replay_child_overlay( Ok(replay.seed) } -async fn replay_suffix( - partition_id: PartitionId, - ownership_epoch: u64, - range: &PartitionRange, - checkpoint: &Checkpoint, - retained_results: usize, - tree: &dyn PartitionTree, - journal: &dyn PartitionJournal, -) -> Result { - const WINDOW_BYTES: usize = 1024 * 1024; - let tail = journal.tail(); - if checkpoint.replay_offset > tail { - return Err(ChunkKvError::JournalCorruption( - "checkpoint replay offset exceeds journal tail".into(), - )); - } - let mut replay = ReplayState::new(checkpoint, retained_results); - let mut read_offset = checkpoint.replay_offset; - let mut frame_offset = read_offset; - let mut buffered = BytesMut::new(); - while read_offset < tail { - let bytes = journal.read_window(read_offset, WINDOW_BYTES).await?; - if bytes.is_empty() { - return Err(ChunkKvError::JournalCorruption( - "journal returned no bytes before durable tail".into(), - )); - } - read_offset = read_offset - .checked_add(bytes.len() as u64) - .ok_or_else(|| ChunkKvError::JournalCorruption("journal replay offset overflows".into()))?; - buffered.extend_from_slice(&bytes); - loop { - let decoded = decode_frame(&buffered)?; - let FrameDecode::Complete(decoded) = decoded else { - break; - }; - validate_replay_record(partition_id, ownership_epoch, &decoded.record)?; - let belongs = range.contains(decoded.record.operation.key()); - replay - .process(tree, frame_offset, decoded.record, belongs) - .await?; - let consumed = decoded.bytes_consumed; - buffered.advance(consumed); - frame_offset = frame_offset - .checked_add(consumed as u64) - .ok_or_else(|| ChunkKvError::JournalCorruption("journal frame offset overflows".into()))?; - } - if buffered.len() > crate::MAX_FRAME_BYTES { - return Err(ChunkKvError::JournalCorruption( - "incomplete frame exceeds maximum size".into(), - )); - } - } - if !buffered.is_empty() { - return Err(ChunkKvError::IncompleteFrame); - } - Ok(replay.seed) -} - -impl ReplayState { - fn new(checkpoint: &Checkpoint, retained_results: usize) -> Self { - Self { - seed: RecoverySeed { - applied_seq: checkpoint.applied_seq, - applied_position: checkpoint.replay_offset, - retry_replay_offset: checkpoint.replay_offset, - results: HashMap::new(), - result_order: std::collections::VecDeque::new(), - expired_floor: HashMap::new(), - recovered: true, - }, - checkpoint_applied_seq: checkpoint.applied_seq, - stream_name: checkpoint.stream_name, - retained_results, - replayed: HashMap::new(), - last_new_sequence: None, - } - } - - async fn process( - &mut self, - tree: &dyn PartitionTree, - frame_offset: u64, - record: WalRecord, - belongs_to_partition: bool, - ) -> Result<()> { - if let Some(previous) = self.replayed.get(&record.mutation_seq) { - if previous != &record { - return Err(ChunkKvError::JournalCorruption( - "conflicting duplicate mutation sequence".into(), - )); - } - return Ok(()); - } - if let Some(last) = self.last_new_sequence { - let expected = last.checked_add(1).ok_or_else(|| { - ChunkKvError::JournalCorruption("journal mutation sequence overflows".into()) - })?; - if expected != record.mutation_seq { - return Err(ChunkKvError::JournalCorruption(format!( - "journal mutation sequence has a gap: expected {expected}, got {} at offset {frame_offset}", - record.mutation_seq - ))); - } - } - if record.mutation_seq > self.checkpoint_applied_seq { - let expected = self - .seed - .applied_seq - .checked_add(1) - .ok_or_else(|| ChunkKvError::JournalCorruption("mutation sequence overflows".into()))?; - if record.mutation_seq != expected { - return Err(ChunkKvError::JournalCorruption(format!( - "journal mutation sequence has a gap after checkpoint {}: expected {expected}, got {} at offset {frame_offset}", - self.checkpoint_applied_seq, record.mutation_seq - ))); - } - if belongs_to_partition && record.result.applied() { - tree.apply(record.mutation_seq, &record.operation) - .await - .map_err(|error| { - ChunkKvError::Internal(format!( - "replay tree apply failed at mutation sequence {}: {error}", - record.mutation_seq - )) - })?; - } else { - tree.advance_noop(record.mutation_seq).await.map_err(|error| { - ChunkKvError::Internal(format!( - "replay tree no-op failed at mutation sequence {}: {error}", - record.mutation_seq - )) - })?; - } - self.seed.applied_seq = record.mutation_seq; - } - self.seed.applied_position = frame_offset; - if belongs_to_partition { - let response = MutationResponse { - mutation_seq: record.mutation_seq, - result: record.result.clone(), - journal_position: JournalPosition { - stream_name: self.stream_name, - offset: frame_offset, - }, - }; - retain_recovered( - &mut self.seed, - self.retained_results, - record.request_id, - record.operation_digest, - response, - )?; - } - self.last_new_sequence = Some(record.mutation_seq); - self.replayed.insert(record.mutation_seq, record); - Ok(()) - } -} - -fn validate_replay_record(partition_id: PartitionId, ownership_epoch: u64, record: &WalRecord) -> Result<()> { - if record.partition_id != partition_id || record.ownership_epoch > ownership_epoch { - return Err(ChunkKvError::JournalCorruption( - "WAL record partition or epoch is invalid".into(), - )); - } - if record.operation_digest != canonical_operation_digest(&record.operation) { - return Err(ChunkKvError::JournalCorruption( - "WAL operation digest mismatch".into(), - )); - } - match record.result { - MutationResult::Applied { revision } if revision != record.mutation_seq => Err( - ChunkKvError::JournalCorruption("applied revision differs from mutation sequence".into()), - ), - _ => Ok(()), - } -} - fn retain_recovered( seed: &mut RecoverySeed, retained_results: usize, diff --git a/lib/crowdb-chunk-kv/src/partition/native.rs b/lib/crowdb-chunk-kv/src/partition/native.rs new file mode 100644 index 000000000..ec71219f2 --- /dev/null +++ b/lib/crowdb-chunk-kv/src/partition/native.rs @@ -0,0 +1,81 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Blocking native storage construction for partition recovery. + +use super::{tree, CrowdbPartitionTree, PartitionJournal, PartitionTree, StreamPartitionJournal}; +use crate::{Checkpoint, ChunkKvError, PartitionRange, Result}; +use crowdb_chunk_stream::ChunkStream; +use std::sync::Arc; + +pub(super) fn native_storage_parts( + tree_id: u64, + range: &PartitionRange, + mut tree_config: crowdb_tree_ffi::Config, + page_store: Arc, + stream: ChunkStream, +) -> Result<(Arc, Arc)> { + tree_config.page_store = Some(page_store); + tree_config.key_range = crowdb_tree_ffi::KeyRange::Bounded { + start: range.start.clone(), + end: range.end.clone(), + }; + let tree: Arc = Arc::new(CrowdbPartitionTree::open(tree_id, &tree_config)?); + let stream_name = stream.stream_name(); + let journal: Arc = Arc::new(StreamPartitionJournal::new(stream, stream_name)?); + Ok((tree, journal)) +} + +pub(super) async fn open_storage( + tree_id: u64, + range: PartitionRange, + tree_config: crowdb_tree_ffi::Config, + page_store: Arc, + stream: ChunkStream, +) -> Result<(Arc, Arc)> { + tokio::task::spawn_blocking(move || { + native_storage_parts(tree_id, &range, tree_config, page_store, stream) + }) + .await + .map_err(|error| ChunkKvError::TreeUnavailable(error.to_string()))? +} + +pub(super) async fn open_latest( + tree_id: u64, + range: PartitionRange, + tree_config: crowdb_tree_ffi::Config, + page_store: Arc, + stream: ChunkStream, +) -> Result<(Arc, Arc, Checkpoint)> { + tokio::task::spawn_blocking(move || { + let replay_offset = if page_store.is_chunk_backed() && tree::chunk_root_generation(&page_store)? == 0 + { + 0 + } else { + page_store.wal_replay_offset().map_err(|error| match error { + crowdb_tree_ffi::CtError::Corruption => ChunkKvError::TreeCorruption(error.to_string()), + _ => ChunkKvError::TreeUnavailable(error.to_string()), + })? + }; + let stream_name = stream.stream_name(); + let stream_generation = stream.record_generation(); + let (tree, journal) = native_storage_parts(tree_id, &range, tree_config, page_store, stream)?; + let (tree_manifest, applied_seq) = tree.checkpoint_state()?; + let root_manifest_generation = tree.root_manifest_generation()?; + Ok(( + tree, + journal, + Checkpoint { + tree_id, + tree_manifest, + root_manifest_generation, + applied_seq, + stream_name, + stream_generation, + replay_offset, + }, + )) + }) + .await + .map_err(|error| ChunkKvError::TreeUnavailable(error.to_string()))? +} diff --git a/lib/crowdb-chunk-kv/src/partition/replay.rs b/lib/crowdb-chunk-kv/src/partition/replay.rs new file mode 100644 index 000000000..ace9e3ac0 --- /dev/null +++ b/lib/crowdb-chunk-kv/src/partition/replay.rs @@ -0,0 +1,236 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Streaming WAL replay with compact, complete-record duplicate identities. + +use super::{ + canonical_operation_digest, decode_frame, retain_recovered, Buf, BytesMut, Checkpoint, ChunkKvError, + FrameDecode, HashMap, JournalPosition, MutationResponse, MutationResult, PartitionId, PartitionJournal, + PartitionRange, PartitionTree, RecoverySeed, Result, StreamName, WalRecord, +}; +use sha2::{Digest, Sha256}; +use std::io::Write; + +pub(super) struct ReplayState { + pub(super) seed: RecoverySeed, + pub(super) checkpoint_applied_seq: u64, + pub(super) stream_name: StreamName, + pub(super) retained_results: usize, + pub(super) replayed: HashMap, + pub(super) last_new_sequence: Option, + pub(super) applied_bytes_since_drain: usize, +} + +pub(super) async fn replay_suffix( + partition_id: PartitionId, + ownership_epoch: u64, + range: &PartitionRange, + checkpoint: &Checkpoint, + retained_results: usize, + tree: &dyn PartitionTree, + journal: &dyn PartitionJournal, +) -> Result { + const WINDOW_BYTES: usize = 1024 * 1024; + let tail = journal.tail(); + if checkpoint.replay_offset > tail { + return Err(ChunkKvError::JournalCorruption( + "checkpoint replay offset exceeds journal tail".into(), + )); + } + let mut replay = ReplayState::new(checkpoint, retained_results); + let mut read_offset = checkpoint.replay_offset; + let mut frame_offset = read_offset; + let mut buffered = BytesMut::new(); + while read_offset < tail { + let bytes = journal.read_window(read_offset, WINDOW_BYTES).await?; + if bytes.is_empty() { + return Err(ChunkKvError::JournalCorruption( + "journal returned no bytes before durable tail".into(), + )); + } + read_offset = read_offset + .checked_add(bytes.len() as u64) + .ok_or_else(|| ChunkKvError::JournalCorruption("journal replay offset overflows".into()))?; + buffered.extend_from_slice(&bytes); + loop { + let decoded = decode_frame(&buffered)?; + let FrameDecode::Complete(decoded) = decoded else { + break; + }; + validate_replay_record(partition_id, ownership_epoch, &decoded.record)?; + let belongs = range.contains(decoded.record.operation.key()); + replay + .process(tree, frame_offset, decoded.record, belongs) + .await?; + let consumed = decoded.bytes_consumed; + buffered.advance(consumed); + frame_offset = frame_offset + .checked_add(consumed as u64) + .ok_or_else(|| ChunkKvError::JournalCorruption("journal frame offset overflows".into()))?; + } + if buffered.len() > crate::MAX_FRAME_BYTES { + return Err(ChunkKvError::JournalCorruption( + "incomplete frame exceeds maximum size".into(), + )); + } + } + if !buffered.is_empty() { + return Err(ChunkKvError::IncompleteFrame); + } + Ok(replay.seed) +} + +impl ReplayState { + pub(super) fn new(checkpoint: &Checkpoint, retained_results: usize) -> Self { + Self { + seed: RecoverySeed { + applied_seq: checkpoint.applied_seq, + applied_position: checkpoint.replay_offset, + retry_replay_offset: checkpoint.replay_offset, + results: HashMap::new(), + result_order: std::collections::VecDeque::new(), + expired_floor: HashMap::new(), + recovered: true, + }, + checkpoint_applied_seq: checkpoint.applied_seq, + stream_name: checkpoint.stream_name, + retained_results, + replayed: HashMap::new(), + last_new_sequence: None, + applied_bytes_since_drain: 0, + } + } + + pub(super) async fn process( + &mut self, + tree: &dyn PartitionTree, + frame_offset: u64, + record: WalRecord, + belongs_to_partition: bool, + ) -> Result<()> { + let fingerprint = record_fingerprint(&record)?; + if let Some(previous) = self.replayed.get(&record.mutation_seq) { + if previous != &fingerprint { + return Err(ChunkKvError::JournalCorruption( + "conflicting duplicate mutation sequence".into(), + )); + } + return Ok(()); + } + if let Some(last) = self.last_new_sequence { + let expected = last.checked_add(1).ok_or_else(|| { + ChunkKvError::JournalCorruption("journal mutation sequence overflows".into()) + })?; + if expected != record.mutation_seq { + return Err(ChunkKvError::JournalCorruption(format!( + "journal mutation sequence has a gap: expected {expected}, got {} at offset {frame_offset}", + record.mutation_seq + ))); + } + } + if record.mutation_seq > self.checkpoint_applied_seq { + let expected = self + .seed + .applied_seq + .checked_add(1) + .ok_or_else(|| ChunkKvError::JournalCorruption("mutation sequence overflows".into()))?; + if record.mutation_seq != expected { + return Err(ChunkKvError::JournalCorruption(format!( + "journal mutation sequence has a gap after checkpoint {}: expected {expected}, got {} at offset {frame_offset}", + self.checkpoint_applied_seq, record.mutation_seq + ))); + } + if belongs_to_partition && record.result.applied() { + tree.apply(record.mutation_seq, &record.operation) + .await + .map_err(|error| { + ChunkKvError::Internal(format!( + "replay tree apply failed at mutation sequence {}: {error}", + record.mutation_seq + )) + })?; + self.applied_bytes_since_drain = self + .applied_bytes_since_drain + // Include arena/index overhead for small values and deletes. + .saturating_add(256) + .saturating_add(record.operation.key().len()) + .saturating_add(record.operation.successful_value().map_or(0, <[u8]>::len)); + if self.applied_bytes_since_drain >= 16 * 1024 * 1024 { + tree.drain_replay().await?; + self.applied_bytes_since_drain = 0; + } + } else { + tree.advance_noop(record.mutation_seq).await.map_err(|error| { + ChunkKvError::Internal(format!( + "replay tree no-op failed at mutation sequence {}: {error}", + record.mutation_seq + )) + })?; + } + self.seed.applied_seq = record.mutation_seq; + } + self.seed.applied_position = frame_offset; + if belongs_to_partition { + let response = MutationResponse { + mutation_seq: record.mutation_seq, + result: record.result.clone(), + journal_position: JournalPosition { + stream_name: self.stream_name, + offset: frame_offset, + }, + }; + retain_recovered( + &mut self.seed, + self.retained_results, + record.request_id, + record.operation_digest, + response, + )?; + } + self.last_new_sequence = Some(record.mutation_seq); + self.replayed.insert(record.mutation_seq, fingerprint); + Ok(()) + } +} + +pub(super) fn validate_replay_record( + partition_id: PartitionId, + ownership_epoch: u64, + record: &WalRecord, +) -> Result<()> { + if record.partition_id != partition_id || record.ownership_epoch > ownership_epoch { + return Err(ChunkKvError::JournalCorruption( + "WAL record partition or epoch is invalid".into(), + )); + } + if record.operation_digest != canonical_operation_digest(&record.operation) { + return Err(ChunkKvError::JournalCorruption( + "WAL operation digest mismatch".into(), + )); + } + match record.result { + MutationResult::Applied { revision } if revision != record.mutation_seq => Err( + ChunkKvError::JournalCorruption("applied revision differs from mutation sequence".into()), + ), + _ => Ok(()), + } +} + +fn record_fingerprint(record: &WalRecord) -> Result<[u8; 32]> { + struct HashWriter(Sha256); + impl Write for HashWriter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.update(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let mut writer = HashWriter(Sha256::new()); + writer.0.update(b"crowdb-chunk-kv-replay-record-v1"); + bincode::serialize_into(&mut writer, record).map_err(|error| { + ChunkKvError::JournalCorruption(format!("WAL record fingerprint failed: {error}")) + })?; + Ok(writer.0.finalize().into()) +} diff --git a/lib/crowdb-chunk-kv/src/partition/transfer.rs b/lib/crowdb-chunk-kv/src/partition/transfer.rs index 24a695951..989a00730 100644 --- a/lib/crowdb-chunk-kv/src/partition/transfer.rs +++ b/lib/crowdb-chunk-kv/src/partition/transfer.rs @@ -229,6 +229,7 @@ async fn catch_up_inner(state: &mut super::WorkerState, request: &TransferCatchU let mut replay = super::ReplayState { seed, checkpoint_applied_seq: current.applied_seq, + applied_bytes_since_drain: 0, stream_name: request.parent_journal.stream_name(), retained_results: state.config.retained_results, replayed: HashMap::new(), diff --git a/lib/crowdb-chunk-kv/src/partition/tree.rs b/lib/crowdb-chunk-kv/src/partition/tree.rs index f507dde5b..6dcd17fd0 100644 --- a/lib/crowdb-chunk-kv/src/partition/tree.rs +++ b/lib/crowdb-chunk-kv/src/partition/tree.rs @@ -47,6 +47,12 @@ pub trait PartitionTree: Send + Sync { ) -> Result<(Vec, bool)>; async fn apply(&self, mutation_seq: u64, operation: &MutationOperation) -> Result<()>; async fn advance_noop(&self, mutation_seq: u64) -> Result<()>; + /// Drains replay's transient buffers without publishing a checkpoint or trimming WAL. + /// # Errors + /// Returns an in-memory tree drain or storage-read failure. + async fn drain_replay(&self) -> Result<()> { + Ok(()) + } async fn checkpoint(&self, wal_replay_offset: u64) -> Result<(u64, u64)>; async fn checkpoint_snapshot( &self, @@ -418,6 +424,18 @@ impl PartitionTree for CrowdbPartitionTree { Ok(()) } + async fn drain_replay(&self) -> Result<()> { + let tree = std::sync::Arc::clone(&self.tree); + let runtime = tokio::runtime::Handle::current(); + tokio::task::spawn_blocking(move || { + let writer = crowdb_tree_ffi::AsyncCrowdbtree::from_shared(tree); + runtime.block_on(writer.flush()) + }) + .await + .map_err(|error| ChunkKvError::TreeUnavailable(error.to_string()))? + .map_err(map_tree_apply_error) + } + async fn checkpoint(&self, wal_replay_offset: u64) -> Result<(u64, u64)> { if let Some(page_store) = self.config.as_ref().and_then(|config| config.page_store.as_ref()) { page_store @@ -451,10 +469,13 @@ impl PartitionTree for CrowdbPartitionTree { wal_replay_offset: u64, ) -> Result<(u64, u64, std::sync::Arc)> { let (generation, applied_seq) = self.checkpoint(wal_replay_offset).await?; - let config = self.config.as_ref().ok_or_else(|| { + let config = self.config.clone().ok_or_else(|| { ChunkKvError::InvalidRequest("native tree was not opened from a retained configuration".into()) })?; - let snapshot = Self::open(self.tree_id, config)?; + let tree_id = self.tree_id; + let snapshot = tokio::task::spawn_blocking(move || Self::open(tree_id, &config)) + .await + .map_err(|error| ChunkKvError::TreeUnavailable(error.to_string()))??; let observed = snapshot.tree.snapshot_state().map_err(map_tree_read_error)?; if observed != (generation, applied_seq) { return Err(ChunkKvError::TreeCorruption(format!( @@ -482,13 +503,20 @@ impl PartitionTree for CrowdbPartitionTree { start: range.start.clone(), end: range.end.clone(), }; - let retained_config = config.clone(); - let (tree, stats) = self.tree.rebuild_range(&config).map_err(map_tree_read_error)?; + let source = std::sync::Arc::clone(&self.tree); + let (tree, stats, config) = tokio::task::spawn_blocking(move || { + source + .rebuild_range(&config) + .map(|(tree, stats)| (tree, stats, config)) + }) + .await + .map_err(|error| ChunkKvError::TreeUnavailable(error.to_string()))? + .map_err(map_tree_read_error)?; Ok(( std::sync::Arc::new(Self { tree_id, tree: std::sync::Arc::new(tree), - config: Some(retained_config), + config: Some(config), }), stats, )) diff --git a/lib/crowdb-chunk-kv/tests/common/generated_journal.rs b/lib/crowdb-chunk-kv/tests/common/generated_journal.rs new file mode 100644 index 000000000..13bb72d01 --- /dev/null +++ b/lib/crowdb-chunk-kv/tests/common/generated_journal.rs @@ -0,0 +1,121 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use async_trait::async_trait; +use bytes::Bytes; +use crowdb_chunk_kv::{ + canonical_operation_digest, encode_frame, Checkpoint, ChunkKvError, JournalPosition, MutationOperation, + MutationResult, PartitionId, PartitionJournal, RequestId, Result, WalRecord, +}; +use crowdb_chunk_stream::StreamName; + +pub struct TestGeneratedJournal { + count: u64, + frame_bytes: usize, + duplicate: Option, + request_conflict: bool, +} + +impl TestGeneratedJournal { + pub fn new(count: u64, duplicate: Option, request_conflict: bool) -> Self { + Self { + count, + frame_bytes: encode_frame(&Self::record(1, false)).unwrap().len(), + duplicate, + request_conflict, + } + } + + pub fn checkpoint(&self) -> Checkpoint { + Checkpoint { + tree_id: 1, + tree_manifest: 0, + root_manifest_generation: 0, + applied_seq: 0, + stream_name: self.stream_name(), + stream_generation: 1, + replay_offset: 0, + } + } + + fn record(sequence: u64, changed: bool) -> WalRecord { + let operation = MutationOperation::Put { + key: b"overwritten".to_vec(), + value: vec![if changed { 255 } else { (sequence % 251) as u8 }; 65536], + }; + WalRecord { + partition_id: PartitionId { high: 1, low: 1 }, + ownership_epoch: 1, + mutation_seq: sequence, + request_id: RequestId { + client_high: 1, + client_low: 1, + client_sequence: sequence, + }, + operation_digest: canonical_operation_digest(&operation), + result: MutationResult::Applied { revision: sequence }, + operation, + } + } +} + +#[async_trait] +impl PartitionJournal for TestGeneratedJournal { + async fn append_frames(&self, _: &[Bytes]) -> Result> { + Err(ChunkKvError::Internal("generated journal is read-only".into())) + } + + async fn read_window(&self, offset: u64, max_bytes: usize) -> Result { + let length = usize::try_from(self.tail().saturating_sub(offset)) + .unwrap() + .min(max_bytes); + let mut out = Vec::with_capacity(length); + while out.len() < length { + let position = usize::try_from(offset).unwrap() + out.len(); + let ordinal = position / self.frame_bytes; + let within = position % self.frame_bytes; + let (sequence, changed) = if ordinal as u64 == self.count { + (1, self.duplicate.unwrap()) + } else { + (ordinal as u64 + 1, false) + }; + let mut record = Self::record(sequence, changed); + if ordinal as u64 == self.count && self.request_conflict { + record.request_id.client_low = 2; + } + let frame = encode_frame(&record)?; + assert_eq!(frame.len(), self.frame_bytes); + let count = (frame.len() - within).min(length - out.len()); + out.extend_from_slice(&frame[within..within + count]); + } + Ok(Bytes::from(out)) + } + + async fn trim_prefix(&self, _: u64) -> Result { + Ok(0) + } + + async fn reclaim_metadata_before(&self, _: u64, _: usize) -> Result { + Ok(0) + } + + async fn checkpoint(&self) -> Result { + Ok(1) + } + + async fn close(&self) -> Result<()> { + Ok(()) + } + + fn stream_name(&self) -> StreamName { + StreamName { high: 1, low: 1 } + } + + fn record_generation(&self) -> u64 { + 1 + } + + fn tail(&self) -> u64 { + (self.count + u64::from(self.duplicate.is_some())) * self.frame_bytes as u64 + } +} diff --git a/lib/crowdb-chunk-kv/tests/common/root_catalog.rs b/lib/crowdb-chunk-kv/tests/common/root_catalog.rs new file mode 100644 index 000000000..927a0585e --- /dev/null +++ b/lib/crowdb-chunk-kv/tests/common/root_catalog.rs @@ -0,0 +1,67 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use arc_swap::ArcSwap; +use crowdb_tree_ffi::{CtError, RootCatalogObject, RootCatalogStore}; +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +#[derive(Default)] +pub struct TestRootCatalog { + objects: ArcSwap>>, + next_reference: AtomicU64, + pub armed: AtomicBool, + pub entered: tokio::sync::Notify, + pub async_progress: AtomicBool, + pub observed_progress: AtomicBool, +} + +impl TestRootCatalog { + fn key(tree_id: u64, object: RootCatalogObject) -> (u64, u8, u64) { + match object { + RootCatalogObject::CurrentManifest => (tree_id, 0, 0), + RootCatalogObject::Manifest(generation) => (tree_id, 1, generation), + RootCatalogObject::ReferenceSegment(id) => (tree_id, 2, id), + } + } + + fn wait_for_async_progress(&self) { + if !self.armed.swap(false, Ordering::AcqRel) { + return; + } + self.entered.notify_one(); + let deadline = Instant::now() + Duration::from_secs(1); + while !self.async_progress.load(Ordering::Acquire) && Instant::now() < deadline { + std::thread::yield_now(); + } + self.observed_progress + .store(self.async_progress.load(Ordering::Acquire), Ordering::Release); + } +} + +impl RootCatalogStore for TestRootCatalog { + fn load(&self, tree_id: u64, object: RootCatalogObject) -> Result>, CtError> { + self.wait_for_async_progress(); + Ok(self.objects.load().get(&Self::key(tree_id, object)).cloned()) + } + + fn store(&self, tree_id: u64, object: RootCatalogObject, data: &[u8]) -> Result<(), CtError> { + self.objects.rcu(|previous| { + let mut next = (**previous).clone(); + next.insert(Self::key(tree_id, object), data.to_vec()); + next + }); + Ok(()) + } + + fn publish(&self, tree_id: u64, _: u64, _: u64, generation: u64, manifest: &[u8]) -> Result<(), CtError> { + self.wait_for_async_progress(); + self.store(tree_id, RootCatalogObject::Manifest(generation), manifest)?; + self.store(tree_id, RootCatalogObject::CurrentManifest, manifest) + } + + fn allocate_reference_segment_id(&self, _: u64) -> Result { + Ok(self.next_reference.fetch_add(1, Ordering::AcqRel) + 1) + } +} diff --git a/lib/crowdb-chunk-kv/tests/native_runtime_test.rs b/lib/crowdb-chunk-kv/tests/native_runtime_test.rs new file mode 100644 index 000000000..d21253a94 --- /dev/null +++ b/lib/crowdb-chunk-kv/tests/native_runtime_test.rs @@ -0,0 +1,135 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/root_catalog.rs"] +mod root_catalog; + +use crowdb_chunk_kv::{CrowdbPartitionTree, MutationOperation, PartitionRange, PartitionTree}; +use crowdb_chunk_kv::{Partition, PartitionConfig, PartitionId}; +use crowdb_chunk_stream::{ + memory::MemoryStreamStore, ChunkStream, StreamConfig, StreamName, StreamPurpose, StreamRecord, + StreamRecordState, +}; +use crowdb_tree_ffi::{ChunkPageStoreOptions, ChunkRootCatalog, Config, PageStore}; +use root_catalog::TestRootCatalog; +use std::sync::atomic::Ordering; +use std::sync::Arc; + +fn config(tree_id: u64, catalog: &Arc) -> Config { + Config { + page_store: Some(Arc::new( + PageStore::open_chunk( + ChunkPageStoreOptions { + tree_id, + owner_epoch: 1, + open_generation: 0, + pack_bytes: 4096, + iu_size: 1, + max_concurrent_packs: 2, + materialization_bytes_per_pass: 4096, + max_chunk_bytes: 1024 * 1024, + mirror_copies: 0, + }, + catalog.clone(), + None, + ) + .unwrap(), + )), + ..Config::default() + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn range_rebuild_keeps_the_runtime_available_during_catalog_io() { + let backend = Arc::new(TestRootCatalog::default()); + let catalog = Arc::new(ChunkRootCatalog::open_callback(backend.clone()).unwrap()); + let source = CrowdbPartitionTree::open(91, &config(91, &catalog)).unwrap(); + source + .apply( + 1, + &MutationOperation::Put { + key: b"key".to_vec(), + value: b"value".to_vec(), + }, + ) + .await + .unwrap(); + source.checkpoint(0).await.unwrap(); + let child_config = config(92, &catalog); + let progress = backend.clone(); + let heartbeat = tokio::spawn(async move { + progress.entered.notified().await; + progress.async_progress.store(true, Ordering::Release); + }); + backend.armed.store(true, Ordering::Release); + let (child, _) = tokio::spawn(async move { + source + .rebuild_range(92, &PartitionRange::default(), child_config) + .await + .unwrap() + }) + .await + .unwrap(); + heartbeat.await.unwrap(); + assert!( + backend.observed_progress.load(Ordering::Acquire), + "catalog IO blocked the only async worker" + ); + assert_eq!(child.get(b"key").await.unwrap().unwrap().value, b"value"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn native_recovery_keeps_the_runtime_available_during_catalog_io() { + let backend = Arc::new(TestRootCatalog::default()); + let catalog = Arc::new(ChunkRootCatalog::open_callback(backend.clone()).unwrap()); + let config = config(93, &catalog); + let pages = config.page_store.clone().unwrap(); + let store = Arc::new(MemoryStreamStore::new(4096)); + let stream = ChunkStream::create( + StreamRecord { + purpose: StreamPurpose::Wal, + stream_name: StreamName { high: 1, low: 93 }, + metadata_group_id: 7, + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("chunk-kv-partition".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), + }, + 1, + StreamConfig::default(), + store.clone(), + store.clone(), + store, + ) + .await + .unwrap(); + let progress = backend.clone(); + let heartbeat = tokio::spawn(async move { + progress.entered.notified().await; + progress.async_progress.store(true, Ordering::Release); + }); + backend.armed.store(true, Ordering::Release); + let recovered = tokio::spawn(async move { + Partition::recover_native_latest_prepared_assignment( + PartitionId { high: 1, low: 93 }, + PartitionRange::default(), + 1, + 93, + PartitionConfig::default(), + config, + pages, + stream, + ) + .await + .unwrap() + }) + .await + .unwrap(); + heartbeat.await.unwrap(); + assert!( + backend.observed_progress.load(Ordering::Acquire), + "native recovery blocked the only async worker" + ); + assert_eq!(recovered.snapshot().applied_seq, 0); +} diff --git a/lib/crowdb-chunk-kv/tests/replay_memory_test.rs b/lib/crowdb-chunk-kv/tests/replay_memory_test.rs new file mode 100644 index 000000000..8e0538f5a --- /dev/null +++ b/lib/crowdb-chunk-kv/tests/replay_memory_test.rs @@ -0,0 +1,80 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use crowdb_chunk_kv::{ + memory::MemoryPartitionTree, ChunkKvError, Partition, PartitionConfig, PartitionId, PartitionRange, +}; + +#[path = "common/generated_journal.rs"] +mod journal; + +async fn recover(count: u64, duplicate: Option) -> crowdb_chunk_kv::Result { + recover_journal(journal::TestGeneratedJournal::new(count, duplicate, false)).await +} + +async fn recover_journal(journal: journal::TestGeneratedJournal) -> crowdb_chunk_kv::Result { + let journal = Arc::new(journal); + Partition::recover( + PartitionId { high: 1, low: 1 }, + PartitionRange::default(), + 1, + journal.checkpoint(), + PartitionConfig::default(), + Arc::new(MemoryPartitionTree::default()), + journal, + ) + .await +} + +#[cfg(target_os = "linux")] +fn peak_rss_kib() -> u64 { + std::fs::read_to_string("/proc/self/status") + .unwrap() + .lines() + .find_map(|line| line.strip_prefix("VmHWM:")) + .unwrap() + .split_whitespace() + .next() + .unwrap() + .parse() + .unwrap() +} + +#[cfg(target_os = "linux")] +#[tokio::test] +async fn replay_does_not_retain_half_a_gibibyte_of_overwritten_payloads() { + let before = peak_rss_kib(); + let partition = recover(8192, None).await.unwrap(); + let peak_growth = peak_rss_kib().saturating_sub(before); + println!("WAL replay peak RSS growth: {peak_growth} KiB for 512 MiB of payloads"); + assert!( + peak_growth < 64 * 1024, + "replay retained payload history: {peak_growth} KiB" + ); + let value = partition.get(1, b"overwritten", None).await.unwrap().unwrap(); + assert_eq!(value.revision, 8192); + assert_eq!(value.value, vec![(8192 % 251) as u8; 65536]); +} + +#[tokio::test] +async fn a_late_exact_duplicate_remains_valid() { + let partition = recover(32, Some(false)).await.unwrap(); + assert_eq!(partition.snapshot().applied_seq, 32); +} + +#[tokio::test] +async fn a_late_conflicting_duplicate_is_still_corruption() { + let result = recover(32, Some(true)).await; + assert!(matches!(result, Err(ChunkKvError::JournalCorruption(message)) + if message == "conflicting duplicate mutation sequence")); +} + +#[tokio::test] +async fn identical_operations_with_changed_request_identity_are_conflicting_duplicates() { + let journal = journal::TestGeneratedJournal::new(32, Some(false), true); + let result = recover_journal(journal).await; + assert!(matches!(result, Err(ChunkKvError::JournalCorruption(message)) + if message == "conflicting duplicate mutation sequence")); +} diff --git a/lib/crowdb-chunk-kv/tests/replay_tree_memory_test.rs b/lib/crowdb-chunk-kv/tests/replay_tree_memory_test.rs new file mode 100644 index 000000000..16a3bd85f --- /dev/null +++ b/lib/crowdb-chunk-kv/tests/replay_tree_memory_test.rs @@ -0,0 +1,58 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#![cfg(target_os = "linux")] + +use std::sync::Arc; + +use crowdb_chunk_kv::{CrowdbPartitionTree, Partition, PartitionConfig, PartitionId, PartitionRange}; + +#[path = "common/generated_journal.rs"] +mod journal; + +fn peak_rss_kib() -> u64 { + std::fs::read_to_string("/proc/self/status") + .unwrap() + .lines() + .find_map(|line| line.strip_prefix("VmHWM:")) + .unwrap() + .split_whitespace() + .next() + .unwrap() + .parse() + .unwrap() +} + +#[tokio::test] +async fn native_replay_drains_overwritten_memtables_without_publishing_a_checkpoint() { + let journal = Arc::new(journal::TestGeneratedJournal::new(8192, None, false)); + let config = crowdb_tree_ffi::Config { + page_store: Some(Arc::new(crowdb_tree_ffi::PageStore::open_mem(65536).unwrap())), + ..crowdb_tree_ffi::Config::default() + }; + let tree = Arc::new(CrowdbPartitionTree::open(1, &config).unwrap()); + let before = peak_rss_kib(); + let partition = Partition::recover( + PartitionId { high: 1, low: 1 }, + PartitionRange::default(), + 1, + journal.checkpoint(), + PartitionConfig::default(), + tree, + journal, + ) + .await + .unwrap(); + let peak_growth = peak_rss_kib().saturating_sub(before); + println!("Native tree replay peak RSS growth: {peak_growth} KiB for 512 MiB of overwritten values"); + assert!( + peak_growth < 96 * 1024, + "unflushed replay buffers retained {peak_growth} KiB" + ); + let value = partition.get(1, b"overwritten", None).await.unwrap().unwrap(); + assert_eq!(value.revision, 8192); + assert_eq!(value.value, vec![(8192 % 251) as u8; 65536]); + let observed = partition.observe_tree().unwrap(); + assert_eq!(observed.checkpoint_manifest, 0); + assert_eq!(observed.checkpoint_applied_seq, 0); +} diff --git a/lib/crowdb-chunk-stream/src/memory.rs b/lib/crowdb-chunk-stream/src/memory.rs index 73ac9fc8b..eeddd3563 100644 --- a/lib/crowdb-chunk-stream/src/memory.rs +++ b/lib/crowdb-chunk-stream/src/memory.rs @@ -395,6 +395,10 @@ impl StreamRegistry for MemoryStreamStore { #[async_trait] impl StreamChunkStore for MemoryStreamStore { + fn capacity_limit(&self) -> u64 { + self.chunk_capacity + } + fn bind_writer(&self, _name: StreamName, _epoch: u64, _session: Arc) {} fn mirror_copies(&self) -> u32 { 1 diff --git a/lib/crowdb-chunk-stream/src/production_chunk.rs b/lib/crowdb-chunk-stream/src/production_chunk.rs index 4b7527735..0a9fe25bc 100644 --- a/lib/crowdb-chunk-stream/src/production_chunk.rs +++ b/lib/crowdb-chunk-stream/src/production_chunk.rs @@ -227,6 +227,10 @@ fn update_state(state: &ChunkStateView, chunk: Chunk) { #[async_trait] impl StreamChunkStore for ProductionStreamChunkStore { + fn capacity_limit(&self) -> u64 { + self.chunk_capacity_bytes + } + fn bind_writer(&self, name: StreamName, epoch: u64, session: Arc) { self.owners.insert(name, (epoch, session)); } @@ -234,7 +238,22 @@ impl StreamChunkStore for ProductionStreamChunkStore { self.mirror_copies } async fn current_capacity(&self, chunk_id: ChunkId) -> Result { - let state = self.state(chunk_id).await?; + // Opening a newer checkpoint can reuse an adapter that observed this + // chunk before a different owner grew it. Cached layout is not an + // authoritative bound for that checkpoint. + let chunk = self + .allocator + .query_chunk(QueryChunkRequest { + chunk_id: Some(chunk_id), + }) + .await + .map_err(io_error)? + .chunk + .ok_or_else(|| StreamError::ReadUnavailable("stream chunk is missing".into()))?; + if chunk.id != Some(chunk_id) { + return Err(StreamError::Corruption("stream chunk identity differs".into())); + } + let state = self.install(chunk)?; chunk_capacity(&state.chunk.load()) } async fn allocate_mirrored( @@ -490,7 +509,17 @@ impl StreamChunkStore for ProductionStreamChunkStore { } let chunk = state.chunk.load_full(); let name = StreamName::from_chunk_owner_key(&chunk.owner_key).ok_or(StreamError::StaleWriter)?; - if let Some(context) = self.owner_context(name, writer_epoch)? { + let context = self.owner_context(name, writer_epoch)?; + if chunk.state == ChunkState::Deleted as i32 { + if cursor != 0 || !recovery::finalized_empty_chunk(&chunk) { + return Err(StreamError::Corruption( + "deleted stream chunk is not a proven empty tail".into(), + )); + } + context.ok_or(StreamError::StaleWriter)?.check_io()?; + return Ok(()); + } + if let Some(context) = context { if chunk.state != ChunkState::Sealed as i32 { self.drain_for_seal(&chunk, &context).await?; } diff --git a/lib/crowdb-chunk-stream/src/production_chunk/allocation.rs b/lib/crowdb-chunk-stream/src/production_chunk/allocation.rs index a096ea596..e2e5767b7 100644 --- a/lib/crowdb-chunk-stream/src/production_chunk/allocation.rs +++ b/lib/crowdb-chunk-stream/src/production_chunk/allocation.rs @@ -78,7 +78,9 @@ impl ProductionStreamChunkStore { // Await every submitted synchronization even if another strip failed. let mut pending = FuturesUnordered::new(); let mut strips = strips.iter(); - for strip in strips.by_ref().take(8) { + // Bound mirrored zeroing bursts so initialization leaves transport + // capacity for writes using the already initialized prefix. + for strip in strips.by_ref().take(2) { pending.push(self.initialize_strip(chunk, strip)); } let mut error = None; diff --git a/lib/crowdb-chunk-stream/src/production_chunk/growth.rs b/lib/crowdb-chunk-stream/src/production_chunk/growth.rs index 5250c405c..96dfdc891 100644 --- a/lib/crowdb-chunk-stream/src/production_chunk/growth.rs +++ b/lib/crowdb-chunk-stream/src/production_chunk/growth.rs @@ -13,9 +13,6 @@ use crowdb_protocol::chunkdb::rpc::{ AppendChunkRequest, AppendChunkResponse, Chunk, QueryChunkRequest, Strip, StripType, }; -const MIN_GROWTH_WINDOW_BYTES: u64 = 2 * STREAM_STRIP_BYTES; -const MAX_GROWTH_WINDOW_BYTES: u64 = 8 * STREAM_STRIP_BYTES; - impl ProductionStreamChunkStore { pub(super) async fn grow_active_chunk( &self, @@ -51,16 +48,11 @@ impl ProductionStreamChunkStore { capacity, })); } - // Small appends must not wait for a large initialization burst. - // Large appends amortize allocation over several following writes. - let window = required_capacity - .saturating_sub(write_cursor(&state)) - .saturating_mul(4) - .clamp(MIN_GROWTH_WINDOW_BYTES, MAX_GROWTH_WINDOW_BYTES) - .next_power_of_two(); + // The stream worker has already budgeted its lookahead. Do not + // multiply speculative bytes again at the physical allocation layer. let target = required_capacity - .div_ceil(window) - .saturating_mul(window) + .div_ceil(STREAM_STRIP_BYTES) + .saturating_mul(STREAM_STRIP_BYTES) .min(self.chunk_capacity_bytes); let growth_timer = PhaseTimer::start(&perf().growth); let response = self.request_growth(&chunk, target, capacity).await?; diff --git a/lib/crowdb-chunk-stream/src/production_chunk/recovery.rs b/lib/crowdb-chunk-stream/src/production_chunk/recovery.rs index 49d60f15d..ba4c884b1 100644 --- a/lib/crowdb-chunk-stream/src/production_chunk/recovery.rs +++ b/lib/crowdb-chunk-stream/src/production_chunk/recovery.rs @@ -7,10 +7,12 @@ use super::{io_error, update_state, Ordering, ProductionStreamChunkStore}; use crate::{ActiveChunkDescriptor, RecoveredStreamTail, Result, StreamError, StreamName}; use async_trait::async_trait; use crowdb_chunk_client::{replay_frame_prefix, select_frame_prefix, FrameExtentSink, MirroredFrameReaders}; -use crowdb_protocol::chunkdb::rpc::{ChunkState, QueryChunkRequest}; +use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkState, QueryChunkRequest}; use crowdb_protocol::frame::{ScannedFrameExtent, MAX_FRAME_SCAN_INPUT_BYTES}; use std::{sync::Arc, time::Duration}; +mod drain; + struct RecoveredExtents(Vec); #[async_trait] @@ -45,21 +47,23 @@ impl ProductionStreamChunkStore { { return Err(StreamError::StaleWriter); } + if chunk.state == ChunkState::Deleted as i32 && !finalized_empty_chunk(&chunk) { + return Err(StreamError::Corruption( + "deleted stream chunk lacks empty finalization proof".into(), + )); + } let state = self.install(chunk.clone())?; + if chunk.strips.is_empty() { + let recovered = empty_sealed_tail(&chunk, active)?; + self.verify_recovery_snapshot(&chunk).await?; + context.check_io()?; + update_state(&state, chunk); + return Ok(recovered); + } let deadline = tokio::time::Instant::from_std(context.deadline()) .min(tokio::time::Instant::now() + Duration::from_secs(60)); - for strip in &chunk.strips { - let Some(crowdb_protocol::chunkdb::rpc::Strip::MirrorStrip(mirror)) = &strip.strip else { - return Err(StreamError::Corruption("recovery chunk is not mirrored".into())); - }; - for segment in &mirror.segments { - context.check_io()?; - self.disk_writer - .drain_segment(segment, u64::from(strip.unit_kb) * 1024, deadline.into_std()) - .await - .map_err(io_error)?; - } - } + self.drain_recovery_layout(&chunk, &context, deadline.into_std()) + .await?; let readers = MirroredFrameReaders::new(Arc::new(chunk.clone()), name, self.disk_writer.clone(), deadline) .map_err(|error| observation_error(&error))?; @@ -91,17 +95,7 @@ impl ProductionStreamChunkStore { ) .await .map_err(|error| observation_error(&error))?; - let refreshed = self - .allocator - .query_chunk(QueryChunkRequest { - chunk_id: Some(active.chunk_id), - }) - .await - .map_err(io_error)? - .chunk; - if refreshed.as_ref() != Some(&chunk) { - return Err(StreamError::StaleWriter); - } + self.verify_recovery_snapshot(&chunk).await?; context.check_io()?; update_state(&state, chunk); state @@ -113,6 +107,60 @@ impl ProductionStreamChunkStore { extents: extents.0, }) } + async fn verify_recovery_snapshot(&self, chunk: &Chunk) -> Result<()> { + let refreshed = self + .allocator + .query_chunk(QueryChunkRequest { chunk_id: chunk.id }) + .await + .map_err(io_error)? + .chunk; + if refreshed.as_ref() != Some(chunk) { + return Err(StreamError::StaleWriter); + } + Ok(()) + } +} + +fn empty_sealed_tail(chunk: &Chunk, active: &ActiveChunkDescriptor) -> Result { + use crowdb_protocol::chunk_stream::StreamPurpose; + let valid_purpose = [StreamPurpose::Wal, StreamPurpose::Stream] + .into_iter() + .any(|purpose| purpose.matches(active.chunk_id) && chunk.chunk_type == purpose.chunk_type() as i32); + // Finalization may delete the already sealed empty tail. Its durable zero + // boundary has no physical bytes to scan; no nonempty tombstone is accepted. + if !valid_purpose + || chunk.writer_epoch == 0 + || (chunk.state != ChunkState::Sealed as i32 && !finalized_empty_chunk(chunk)) + || chunk.capacity != 0 + || chunk.sealed_length != 0 + || chunk.acknowledged_cursor != 0 + || active.acknowledged_cursor != 0 + || active.physical_start != 0 + { + return Err(StreamError::Corruption("invalid empty sealed tail".into())); + } + Ok(RecoveredStreamTail { + physical_end: 0, + logical_end: active.logical_start, + extents: Vec::new(), + }) +} + +pub(super) fn finalized_empty_chunk(chunk: &Chunk) -> bool { + use crowdb_protocol::chunk_stream::StreamPurpose; + chunk.state == ChunkState::Deleted as i32 + && chunk.sealed_ts_ms != 0 + && chunk.writer_epoch != 0 + && chunk.strips.is_empty() + && chunk.cleanup_intents.is_empty() + && chunk.capacity == 0 + && chunk.sealed_length == 0 + && chunk.acknowledged_cursor == 0 + && chunk.id.is_some_and(|id| { + [StreamPurpose::Wal, StreamPurpose::Stream] + .into_iter() + .any(|purpose| purpose.matches(id) && chunk.chunk_type == purpose.chunk_type() as i32) + }) } fn observation_error(error: &crowdb_chunk_client::FrameObservationError) -> StreamError { diff --git a/lib/crowdb-chunk-stream/src/production_chunk/recovery/drain.rs b/lib/crowdb-chunk-stream/src/production_chunk/recovery/drain.rs new file mode 100644 index 000000000..e505c4e53 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/production_chunk/recovery/drain.rs @@ -0,0 +1,136 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Failed recovery drains feed the existing durable mirror repair queue. + +use super::{io_error, ProductionStreamChunkStore}; +use crate::{Result, StreamError, StreamWriteContext}; +use crowdb_protocol::{ + chunkdb::rpc::{Chunk, ChunkStrip, ReplaceChunkStripRangeRequest, Strip}, + common::ChunkId, + diskdb::rpc::Segment, +}; +use futures::{stream, StreamExt}; + +type StripDrainResults = Vec<(Segment, Result<()>)>; + +impl ProductionStreamChunkStore { + pub(super) async fn drain_recovery_layout( + &self, + chunk: &Chunk, + context: &StreamWriteContext, + deadline: std::time::Instant, + ) -> Result<()> { + let drained = self.drain_recovery_segments(chunk, context, deadline).await?; + for (index, (strip, results)) in chunk.strips.iter().zip(drained).enumerate() { + let Some(Strip::MirrorStrip(mirror)) = &strip.strip else { + return Err(StreamError::Corruption("recovery chunk is not mirrored".into())); + }; + if !strip.unavailable_segments.is_empty() { + return Err(StreamError::ReadUnavailable( + "recovery mirror repair is pending".into(), + )); + } + let failed: Vec<_> = mirror + .segments + .iter() + .copied() + .filter(|segment| { + results + .iter() + .any(|(observed, result)| observed == segment && result.is_err()) + }) + .collect(); + let Some(error) = results.into_iter().find_map(|(_, result)| result.err()) else { + continue; + }; + context.check_io()?; + // Keep the failed operation unavailable. A drained surviving mirror + // gives the durable repair executor a stable source; its published + // replacement must drain successfully on a later recovery attempt. + if failed.len() < mirror.segments.len() { + tracing::warn!(chunk_id = ?chunk.id, strip_sequence = strip.strip_sequence, + %error, "recovery drain failed; requesting mirror repair"); + self.mark_recovery_mirrors(chunk, index, strip, failed).await?; + } + return Err(error); + } + Ok(()) + } + + async fn drain_recovery_segments( + &self, + chunk: &Chunk, + context: &StreamWriteContext, + deadline: std::time::Instant, + ) -> Result> { + let mut segments = Vec::with_capacity(chunk.strips.len() * 2); + let mut drained = Vec::with_capacity(chunk.strips.len()); + for (index, strip) in chunk.strips.iter().enumerate() { + let Some(Strip::MirrorStrip(mirror)) = &strip.strip else { + return Err(StreamError::Corruption("recovery chunk is not mirrored".into())); + }; + if !strip.unavailable_segments.is_empty() { + return Err(StreamError::ReadUnavailable( + "recovery mirror repair is pending".into(), + )); + } + drained.push(Vec::with_capacity(mirror.segments.len())); + segments.extend( + mirror + .segments + .iter() + .map(|segment| (index, u64::from(strip.unit_kb) * 1024, *segment)), + ); + } + let results = stream::iter(segments) + .map(|(index, unit_bytes, segment)| async move { + let result = async { + context.check_io()?; + self.disk_writer + .drain_segment(&segment, unit_bytes, deadline) + .await + .map_err(io_error) + } + .await; + (index, segment, result) + }) + .buffer_unordered(8) + .collect::>() + .await; + for (index, segment, result) in results { + drained[index].push((segment, result)); + } + Ok(drained) + } + + async fn mark_recovery_mirrors( + &self, + chunk: &Chunk, + index: usize, + strip: &ChunkStrip, + failed: Vec, + ) -> Result<()> { + let id = chunk + .id + .ok_or_else(|| StreamError::Corruption("recovery chunk has no ID".into()))?; + let mut replacement = strip.clone(); + replacement.unavailable_segments = failed; + self.allocator + .replace_chunk_strip_range(ReplaceChunkStripRangeRequest { + chunk_id: Some(id), + expected_modify_ts: chunk.modify_ts, + start_index: u32::try_from(index) + .map_err(|_| StreamError::Corruption("strip index overflow".into()))?, + old_strips: vec![strip.clone()], + replacement_strips: vec![replacement], + operation_id: Some(ChunkId { + high: id.high ^ chunk.modify_ts ^ 0xd7a1_0000_0000_0001, + low: id.low ^ u64::from(strip.strip_sequence), + }), + }) + .await + .map_err(io_error)?; + Ok(()) + } +} diff --git a/lib/crowdb-chunk-stream/src/storage.rs b/lib/crowdb-chunk-stream/src/storage.rs index 9640a4f15..d5570218c 100644 --- a/lib/crowdb-chunk-stream/src/storage.rs +++ b/lib/crowdb-chunk-stream/src/storage.rs @@ -74,6 +74,11 @@ pub trait StreamMetadataStore: Send + Sync { #[async_trait] pub trait StreamChunkStore: Send + Sync { + /// Maximum physical capacity supported by one chunk in this store. + fn capacity_limit(&self) -> u64 { + crowdb_chunk_client::STREAM_CHUNK_BYTES + } + /// Required synchronized replica count for append traffic accounting. fn mirror_copies(&self) -> u32; fn bind_writer(&self, name: StreamName, epoch: u64, session: std::sync::Arc); diff --git a/lib/crowdb-chunk-stream/src/stream/capacity_plan.rs b/lib/crowdb-chunk-stream/src/stream/capacity_plan.rs index bf9ce1026..30e20c12a 100644 --- a/lib/crowdb-chunk-stream/src/stream/capacity_plan.rs +++ b/lib/crowdb-chunk-stream/src/stream/capacity_plan.rs @@ -5,17 +5,20 @@ use std::time::Duration; -const MIN_AHEAD: u64 = 2 * 1024 * 1024; +const MIN_AHEAD: u64 = crowdb_protocol::frame::MAX_FRAME_BYTES as u64; +const MAX_AHEAD: u64 = crowdb_chunk_client::STREAM_STRIP_BYTES; const INITIAL_PREPARATION: Duration = Duration::from_millis(20); #[derive(Default)] pub(super) struct CapacityPlan { bytes_per_second: u64, preparation: Duration, + last_batch_bytes: u64, } impl CapacityPlan { pub(super) fn record_get(&mut self, bytes: u64, elapsed: Duration) { + self.last_batch_bytes = bytes; let rate = u64::try_from(u128::from(bytes) * 1_000_000_000 / elapsed.as_nanos().max(1)).unwrap_or(u64::MAX); // React immediately to acceleration; decay gradually after a slow write. @@ -31,18 +34,7 @@ impl CapacityPlan { if capacity >= limit { return None; } - // Cover twice the observed preparation time, including allocation and every - // replica's initialization. Bound speculative capacity independently of - // stream length and leave room for the batch already being submitted. - let latency = self.preparation.max(INITIAL_PREPARATION).as_nanos(); - let ahead = u64::try_from( - u128::from(self.bytes_per_second) - .saturating_mul(latency) - .saturating_mul(2) - / 1_000_000_000, - ) - .unwrap_or(u64::MAX) - .clamp(MIN_AHEAD, limit.max(MIN_AHEAD)); + let ahead = self.ahead(limit); if capacity.saturating_sub(cursor) > ahead.saturating_add(batch_bytes) { return None; } @@ -54,6 +46,34 @@ impl CapacityPlan { .min(limit), ) } + + pub(super) fn successor_target(&self, cursor: u64, limit: u64) -> Option { + let ahead = self.ahead(limit); + if limit.saturating_sub(cursor) > ahead.saturating_add(self.last_batch_bytes) { + return None; + } + Some( + ahead + .saturating_mul(2) + .saturating_add(self.last_batch_bytes) + .min(limit), + ) + } + + fn ahead(&self, limit: u64) -> u64 { + // Cover twice the observed preparation time, including allocation and every + // replica's initialization. Bound speculative capacity independently of + // stream length and leave room for the batch already being submitted. + let latency = self.preparation.max(INITIAL_PREPARATION).as_nanos(); + u64::try_from( + u128::from(self.bytes_per_second) + .saturating_mul(latency) + .saturating_mul(2) + / 1_000_000_000, + ) + .unwrap_or(u64::MAX) + .clamp(MIN_AHEAD, limit.clamp(MIN_AHEAD, MAX_AHEAD)) + } } #[cfg(feature = "test-util")] diff --git a/lib/crowdb-chunk-stream/src/stream/successor.rs b/lib/crowdb-chunk-stream/src/stream/successor.rs index 4436bbac7..8e84958cf 100644 --- a/lib/crowdb-chunk-stream/src/stream/successor.rs +++ b/lib/crowdb-chunk-stream/src/stream/successor.rs @@ -19,6 +19,22 @@ pub(super) struct SuccessorPreparation { } pub(super) fn start(state: &mut WorkerState) { + let Some(active) = state.progress.active.as_ref() else { + return; + }; + let Some(required) = state.capacity_plan.successor_target( + active.acknowledged_cursor, + state + .config + .chunk_capacity_bytes + .min(state.chunks.capacity_limit()), + ) else { + return; + }; + prepare(state, required); +} + +fn prepare(state: &mut WorkerState, required: u64) { let Some(authority) = state.authority.as_ref() else { return; }; @@ -35,15 +51,7 @@ pub(super) fn start(state: &mut WorkerState) { epoch: state.writer_epoch, purpose: state.progress.purpose, interval: state.config.liveness_interval, - required: state - .capacity_plan - .target( - 0, - 0, - state.config.chunk_capacity_bytes, - state.config.batch_bytes as u64, - ) - .expect("nonzero chunk capacity"), + required, }; let (handoff, receive) = oneshot::channel(); state.successor = Some(SuccessorPreparation { @@ -200,7 +208,9 @@ async fn allocate( pub(super) async fn take(state: &mut WorkerState) -> Result { let Some(preparation) = state.successor.take() else { - start(state); + // A waiting rotation requires a candidate regardless of the last + // consumption sample. Grow it further only for the actual next append. + prepare(state, 1); let preparation = state.successor.take().ok_or(StreamError::WriteStalled)?; return join(preparation, true).await; }; diff --git a/lib/crowdb-chunk-stream/src/stream/worker.rs b/lib/crowdb-chunk-stream/src/stream/worker.rs index bfdeecfed..e231a8823 100644 --- a/lib/crowdb-chunk-stream/src/stream/worker.rs +++ b/lib/crowdb-chunk-stream/src/stream/worker.rs @@ -170,6 +170,8 @@ async fn process_append_batch( let result = write_with_recovery(state, &requests, bytes).await; match result { Ok(ranges) => { + preparation::start(state, bytes); + successor::start(state); for (request, range) in requests.into_iter().zip(ranges) { state .queued_bytes diff --git a/lib/crowdb-chunk-stream/tests/capacity_plan_test.rs b/lib/crowdb-chunk-stream/tests/capacity_plan_test.rs index deb0d6dfd..a63e3892f 100644 --- a/lib/crowdb-chunk-stream/tests/capacity_plan_test.rs +++ b/lib/crowdb-chunk-stream/tests/capacity_plan_test.rs @@ -11,19 +11,19 @@ const MIB: u64 = 1024 * 1024; fn bandwidth_and_preparation_latency_increase_the_reserved_capacity() { let plan = |bytes, millis| { capacity_target_for_tests( - &[(bytes, Duration::from_millis(100))], + &[(bytes, Duration::from_millis(125))], Duration::from_millis(millis), 0, - 8 * MIB, + MIB / 4, 256 * MIB, 4096, ) }; - assert_eq!(plan(MIB, 50), None, "a slow writer has ample ready capacity"); - let fast = plan(20 * MIB, 50).unwrap(); - let slower_preparation = plan(20 * MIB, 100).unwrap(); - assert_eq!(fast, 40 * MIB + 4096); - assert_eq!(slower_preparation, 80 * MIB + 4096); + assert_eq!(plan(4096, 50), None, "a slow writer has ample ready capacity"); + let fast = plan(MIB / 8, 250).unwrap(); + let slower_preparation = plan(MIB / 8, 500).unwrap(); + assert_eq!(fast, MIB + 4096); + assert_eq!(slower_preparation, 2 * MIB + 4096); } #[test] @@ -38,20 +38,21 @@ fn refill_starts_with_two_preparation_intervals_still_ready() { 4096, ) }; - assert!(request(21 * MIB + 4096 + 1).is_none()); - assert!(request(21 * MIB + 4096).is_some()); + assert!(request(2 * MIB + 4096 + 1).is_none()); + assert!(request(2 * MIB + 4096).is_some()); } #[test] fn cold_start_extreme_rates_and_chunk_end_remain_bounded() { assert_eq!( capacity_target_for_tests(&[], Duration::ZERO, 0, MIB, 256 * MIB, 4096), - Some(4 * MIB + 4096) + None, + "a cold stream must consume its first strip before speculative growth" ); let writes = [(u64::MAX, Duration::ZERO)]; assert_eq!( capacity_target_for_tests(&writes, Duration::MAX, MIB, 2 * MIB, 256 * MIB, 4096), - Some(256 * MIB) + Some(3 * MIB + 4096) ); assert_eq!( capacity_target_for_tests(&writes, Duration::MAX, MIB, 256 * MIB, 256 * MIB, 4096), @@ -68,13 +69,14 @@ fn one_slow_write_does_not_remove_a_fast_writers_margin() { ], Duration::from_millis(50), 0, - 8 * MIB, + MIB, 256 * MIB, 4096, ) .unwrap(); - assert!( - target >= 35 * MIB, - "a slow sample must not immediately shrink the reserve" + assert_eq!( + target, + 2 * MIB + 4096, + "one slow sample preserves the bounded reserve" ); } diff --git a/lib/crowdb-chunk-stream/tests/common/initialization_window.rs b/lib/crowdb-chunk-stream/tests/common/initialization_window.rs new file mode 100644 index 000000000..a5532676e --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/common/initialization_window.rs @@ -0,0 +1,81 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crate::Disks; +use async_trait::async_trait; +use bytes::Bytes; +use crowdb_chunk_client::{DiskWriter, IoError}; +use crowdb_protocol::diskdb::rpc::Segment; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +pub(crate) struct TestInitializationWindow { + pub(crate) inner: Arc, + limit: usize, + active: AtomicUsize, + rejected: AtomicUsize, +} + +impl TestInitializationWindow { + pub(crate) fn new(inner: Arc, limit: usize) -> Self { + Self { + inner, + limit, + active: AtomicUsize::new(0), + rejected: AtomicUsize::new(0), + } + } + + pub(crate) fn rejected(&self) -> usize { + self.rejected.load(Ordering::Acquire) + } +} + +#[async_trait] +impl DiskWriter for TestInitializationWindow { + async fn write( + &self, + segment: &Segment, + unit_bytes: u64, + data: Bytes, + ) -> crowdb_chunk_client::Result<()> { + self.inner.write(segment, unit_bytes, data).await + } + + async fn write_at_byte_offset( + &self, + segment: &Segment, + unit_bytes: u64, + offset: u64, + data: Bytes, + ) -> crowdb_chunk_client::Result<()> { + self.inner + .write_at_byte_offset(segment, unit_bytes, offset, data) + .await + } + + async fn initialize_synchronized_at_byte_offset( + &self, + segment: &Segment, + unit_bytes: u64, + offset: u64, + data: Bytes, + ) -> crowdb_chunk_client::Result<()> { + let active = self.active.fetch_add(1, Ordering::AcqRel) + 1; + let result = if active > self.limit { + self.rejected.fetch_add(1, Ordering::AcqRel); + Err(IoError::WriteFailed( + "bounded initialization transport is full".into(), + )) + } else { + tokio::task::yield_now().await; + self.inner + .initialize_synchronized_at_byte_offset(segment, unit_bytes, offset, data) + .await + }; + self.active.fetch_sub(1, Ordering::AcqRel); + result + } +} diff --git a/lib/crowdb-chunk-stream/tests/lease_fenced_release_test.rs b/lib/crowdb-chunk-stream/tests/lease_fenced_release_test.rs new file mode 100644 index 000000000..8d3464442 --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/lease_fenced_release_test.rs @@ -0,0 +1,222 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use async_trait::async_trait; +use crowdb_chunk_stream::{ + memory::MemoryStreamStore, ChunkStream, Result, StreamConfig, StreamError, StreamName, + StreamOwnerAuthority, StreamPurpose, StreamRecord, StreamRecordState, StreamRegistry, + StreamWriterSession, +}; +use std::{ + sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Arc, + }, + time::{Duration, Instant}, +}; + +const HOUR: Duration = Duration::from_secs(3600); + +struct TestAuthority { + deadline: Instant, + fenced: AtomicBool, + releases: AtomicUsize, +} + +#[async_trait] +impl StreamOwnerAuthority for TestAuthority { + fn deadline(&self) -> Result { + if self.fenced.load(Ordering::Acquire) { + Err(StreamError::StaleWriter) + } else { + Ok(self.deadline) + } + } + + async fn release(&self) -> Result<()> { + self.releases.fetch_add(1, Ordering::AcqRel); + Ok(()) + } +} + +struct TestRegistry { + store: Arc, + session: Arc, +} + +#[async_trait] +impl StreamRegistry for TestRegistry { + async fn load(&self, name: StreamName) -> Result> { + self.store.load(name).await + } + + async fn create(&self, record: StreamRecord) -> Result<()> { + self.store.create(record).await + } + + async fn claim_writer(&self, _name: StreamName, _epoch: u64) -> Result> { + Ok(self.session.clone()) + } +} + +struct TestIdleWriter { + stream: ChunkStream, + store: Arc, + authority: Arc, + session: Arc, +} + +impl TestIdleWriter { + async fn create(low: u64) -> Self { + // This fixed std::Instant deadline does not advance with Tokio's + // paused clock. Revocation below is explicit, rather than a TTL race. + let authority = Arc::new(TestAuthority { + deadline: Instant::now() + Duration::from_secs(24 * 3600), + fenced: AtomicBool::new(false), + releases: AtomicUsize::new(0), + }); + let session = StreamWriterSession::new(authority.clone()); + // An empty 8 MiB chunk exceeds the initial lookahead, so no successor + // preparation can contribute a liveness attempt or release error. + let store = Arc::new(MemoryStreamStore::new(8 * 1024 * 1024)); + let registry = Arc::new(TestRegistry { + store: store.clone(), + session: session.clone(), + }); + let stream = ChunkStream::create( + StreamRecord { + state: StreamRecordState::Active, + ..StreamRecord::creating(StreamName { high: 748, low }, None, StreamPurpose::Wal) + }, + 1, + StreamConfig { + liveness_interval: HOUR, + ..StreamConfig::default() + }, + registry, + store.clone(), + store.clone(), + ) + .await + .unwrap(); + for _ in 0..10 { + tokio::task::yield_now().await; + } + assert_eq!( + store.chunks_for_tests().await.len(), + 1, + "no successor may affect the renewal count" + ); + Self { + stream, + store, + authority, + session, + } + } + + fn assert_no_payload_or_release(&self) { + assert_eq!(self.stream.tail(), 0); + assert_eq!(self.stream.metrics().completed, 0); + assert_eq!(self.store.chunk_write_count(), 0); + assert_eq!(self.store.cursor_advance_count(), 0); + assert_eq!(self.authority.releases.load(Ordering::Acquire), 0); + assert!(!self.session.is_released()); + } +} + +async fn enter_failed_liveness_maintenance(fixture: &TestIdleWriter) { + fixture.store.fail_next_renewals(usize::MAX); + tokio::time::advance(HOUR).await; + wait_for_attempts(&fixture.store, 1).await; + tokio::time::advance(Duration::from_millis(100)).await; + wait_for_attempts(&fixture.store, 2).await; + tokio::time::advance(Duration::from_millis(100)).await; + wait_for_attempts(&fixture.store, 4).await; + assert_eq!(fixture.store.liveness_attempt_count_for_tests(), 4); + fixture.assert_no_payload_or_release(); +} + +async fn wait_for_attempts(store: &MemoryStreamStore, attempts: u64) { + // Runnable bounded yields prevent paused Tokio time from auto-advancing + // to the next hourly timer while waiting for a worker transition. + for _ in 0..1000 { + if store.liveness_attempt_count_for_tests() >= attempts { + return; + } + tokio::task::yield_now().await; + } + panic!("liveness worker did not reach {attempts} attempts"); +} + +#[tokio::test(start_paused = true)] +async fn healthy_idle_writer_releases_without_appending() { + let fixture = TestIdleWriter::create(1).await; + fixture.assert_no_payload_or_release(); + fixture.stream.release_writer().await.unwrap(); + assert_eq!(fixture.authority.releases.load(Ordering::Acquire), 1); + assert!(fixture.session.is_released()); + assert_eq!(fixture.store.liveness_attempt_count_for_tests(), 0); + assert_eq!(fixture.store.chunk_write_count(), 0); +} + +#[tokio::test(start_paused = true)] +async fn failed_idle_liveness_then_revocation_stalls_release_before_owner_release() { + let fixture = TestIdleWriter::create(2).await; + let original_deadline = fixture.authority.deadline; + enter_failed_liveness_maintenance(&fixture).await; + // The renewal task performs only three attempts before notifying the + // worker. With no successor or next hourly tick, attempt four proves the + // real worker is in maintain(), sleeping after its first failed renewal. + assert_eq!(fixture.store.liveness_attempt_count_for_tests(), 4); + fixture.assert_no_payload_or_release(); + fixture.authority.fenced.store(true, Ordering::Release); + let stream = fixture.stream.clone(); + let release = tokio::spawn(async move { stream.release_writer().await }); + tokio::task::yield_now().await; + assert!(!release.is_finished(), "maintain must still own the worker"); + tokio::time::advance(Duration::from_millis(100)).await; + for _ in 0..1000 { + if release.is_finished() { + break; + } + tokio::task::yield_now().await; + } + assert!( + release.is_finished(), + "release command did not complete after fencing" + ); + assert_eq!(release.await.unwrap(), Err(StreamError::WriteStalled)); + fixture.assert_no_payload_or_release(); + assert_eq!(fixture.authority.deadline, original_deadline); + assert_eq!(fixture.store.liveness_attempt_count_for_tests(), 4); +} + +#[tokio::test(start_paused = true)] +async fn transient_idle_liveness_failure_recovers_before_release_with_live_authority() { + let fixture = TestIdleWriter::create(3).await; + enter_failed_liveness_maintenance(&fixture).await; + fixture.store.fail_next_renewals(0); + let stream = fixture.stream.clone(); + let release = tokio::spawn(async move { stream.release_writer().await }); + tokio::task::yield_now().await; + assert!(!release.is_finished(), "maintain must still own the worker"); + tokio::time::advance(Duration::from_millis(100)).await; + for _ in 0..1000 { + if release.is_finished() { + break; + } + tokio::task::yield_now().await; + } + assert!( + release.is_finished(), + "release did not complete after liveness recovered" + ); + release.await.unwrap().unwrap(); + assert_eq!(fixture.store.liveness_attempt_count_for_tests(), 5); + assert_eq!(fixture.authority.releases.load(Ordering::Acquire), 1); + assert!(fixture.session.is_released()); + assert!(!fixture.authority.fenced.load(Ordering::Acquire)); + assert_eq!(fixture.store.chunk_write_count(), 0); + assert_eq!(fixture.store.cursor_advance_count(), 0); +} diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test.rs index 89ebc171a..16272acd4 100644 --- a/lib/crowdb-chunk-stream/tests/production_chunk_test.rs +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test.rs @@ -25,15 +25,27 @@ use crowdb_protocol::chunkdb::rpc::{ use crowdb_protocol::common::{ChunkId, DiskId}; use crowdb_protocol::diskdb::rpc::Segment; +#[path = "production_chunk_test/finalized_empty.rs"] +mod finalized_empty; + #[path = "production_chunk_test/recovery.rs"] mod recovery; +#[path = "production_chunk_test/recovery_drain.rs"] +mod recovery_drain; + #[path = "production_chunk_test/local_progress.rs"] mod local_progress; #[path = "production_chunk_test/growth.rs"] mod growth; +#[path = "production_chunk_test/capacity_refresh.rs"] +mod capacity_refresh; + +#[path = "common/initialization_window.rs"] +mod initialization_window; + #[path = "production_chunk_test/preparation.rs"] mod preparation; @@ -441,6 +453,7 @@ struct Disks { initialization_started: tokio::sync::Notify, initialization_resume: tokio::sync::Notify, drain_delay_ms: AtomicU64, + fail_all_drains: AtomicBool, active_drains: AtomicUsize, max_active_drains: AtomicUsize, completed_drains: AtomicUsize, @@ -482,7 +495,11 @@ impl DiskWriter for Disks { if delay != 0 { tokio::time::sleep(std::time::Duration::from_millis(delay)).await; } - let result = self.fsync(segment).await; + let result = if self.fail_all_drains.load(Ordering::Acquire) { + Err(IoError::WriteFailed("all mirror drains unavailable".into())) + } else { + self.fsync(segment).await + }; self.active_drains.fetch_sub(1, Ordering::AcqRel); self.completed_drains.fetch_add(1, Ordering::AcqRel); result @@ -643,7 +660,7 @@ async fn production_store_grows_and_writes_across_mirror_strips() { .unwrap(); assert_eq!(active.capacity, 1024 * 1024); let grown = store - .grow_mirrored(name, 9, active.chunk_id, active.capacity + 1) + .grow_mirrored(name, 9, active.chunk_id, 8 * 1024 * 1024) .await .unwrap() .unwrap(); @@ -867,7 +884,10 @@ async fn stream_repair_preserves_a_batch_crossing_mirror_strips() { created_at_ms: StreamRecord::creation_time_ms(), }, 9, - StreamConfig::default(), + StreamConfig { + chunk_capacity_bytes: 2 * 1024 * 1024, + ..StreamConfig::default() + }, Arc::clone(&metadata) as Arc, metadata as Arc, chunks, @@ -888,7 +908,7 @@ async fn stream_repair_preserves_a_batch_crossing_mirror_strips() { assert_eq!(&read[prefix.len()..], suffix.as_ref()); assert_eq!(stream.metrics().rollovers, 0); let chunk = allocator.chunk.lock().unwrap().clone().unwrap(); - assert_eq!(chunk.strips.len(), 8); + assert_eq!(chunk.strips.len(), 2); let Some(Strip::MirrorStrip(mirror)) = &chunk.strips[1].strip else { panic!("expected mirror strip"); }; diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/capacity_refresh.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/capacity_refresh.rs new file mode 100644 index 000000000..53bc47def --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/capacity_refresh.rs @@ -0,0 +1,77 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; +use crowdb_chunk_stream::{StreamError, StreamPurpose}; + +#[tokio::test] +async fn reopening_refreshes_cached_capacity_after_another_owner_grows_the_chunk() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let make_chunks = || { + Arc::new( + ProductionStreamChunkStore::new( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + ) + .unwrap(), + ) + }; + let metadata = Arc::new(MemoryStreamStore::new(1)); + let name = StreamName { high: 277, low: 281 }; + let writer = ChunkStream::create( + StreamRecord { + metadata_group_id: 7, + state: StreamRecordState::Active, + ..StreamRecord::creating(name, None, StreamPurpose::Stream) + }, + 9, + StreamConfig::default(), + metadata.clone(), + metadata.clone(), + make_chunks(), + ) + .await + .unwrap(); + let chunk_id = allocator.chunk.lock().unwrap().as_ref().unwrap().id.unwrap(); + let observer = make_chunks(); + assert_eq!(observer.current_capacity(chunk_id).await.unwrap(), 1024 * 1024); + let payload = Bytes::from(vec![0x69; 1536 * 1024]); + writer.append(std::slice::from_ref(&payload)).await.unwrap(); + writer.checkpoint().await.unwrap(); + writer.release_writer().await.unwrap(); + let reader = ChunkStream::open_read_only( + name, + 9, + StreamConfig::default(), + metadata.clone(), + metadata.clone(), + observer.clone(), + ) + .await + .expect("a retained adapter must refresh capacity before validating the newer checkpoint"); + assert_eq!(reader.read_at(0, payload.len()).await.unwrap(), payload); + allocator.fail_query_once.store(true, Ordering::Release); + assert!(observer.current_capacity(chunk_id).await.is_err()); + { + let mut guard = allocator.chunk.lock().unwrap(); + let chunk = guard.as_mut().unwrap(); + chunk.strips.truncate(1); + chunk.capacity = 1024; + chunk.modify_ts += 1; + } + let result = ChunkStream::open_read_only( + name, + 9, + StreamConfig::default(), + metadata.clone(), + metadata, + observer, + ) + .await; + assert!( + matches!(result, Err(StreamError::Corruption(message)) if message.contains("checkpoint exceeds chunk capacity")) + ); +} diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/finalized_empty.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/finalized_empty.rs new file mode 100644 index 000000000..ccd9f195a --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/finalized_empty.rs @@ -0,0 +1,145 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; + +#[tokio::test] +async fn finalized_empty_tail_recovers_without_reviving_the_deleted_chunk() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let make = || { + ProductionStreamChunkStore::new( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + ) + .unwrap() + }; + let name = StreamName { high: 813, low: 99 }; + let writer = make(); + let mut active = writer + .allocate_mirrored(name, 9, crowdb_chunk_stream::StreamPurpose::Wal) + .await + .unwrap(); + active.logical_start = 42; + let allocated = allocator.chunk.lock().unwrap().clone().unwrap(); + let deleted = { + let mut guard = allocator.chunk.lock().unwrap(); + let chunk = guard.as_mut().unwrap(); + chunk.state = ChunkState::Deleted as i32; + chunk.sealed_ts_ms = 123; + chunk.modify_ts += 2; + chunk.capacity = 0; + chunk.strips.clear(); + chunk.clone() + }; + drop(writer); + let cold = make(); + assert!(matches!( + cold.seal(active.chunk_id, 10, 0).await, + Err(crowdb_chunk_stream::StreamError::StaleWriter) + )); + let metadata = MemoryStreamStore::new(1024); + cold.bind_writer(name, 10, metadata.claim_writer(name, 10).await.unwrap()); + let recovered = cold.recover_tail(name, 10, &active).await.unwrap(); + assert_eq!((recovered.physical_end, recovered.logical_end), (0, 42)); + assert!(recovered.extents.is_empty()); + cold.seal(active.chunk_id, 10, 0).await.unwrap(); + assert_eq!(*allocator.chunk.lock().unwrap(), Some(deleted.clone())); + for field in 0..11 { + let mut invalid = deleted.clone(); + let mut descriptor = active.clone(); + match field { + 0 => invalid.sealed_ts_ms = 0, + 1 => invalid.capacity = 1, + 2 => invalid.sealed_length = 1, + 3 => invalid.acknowledged_cursor = 1, + 4 => descriptor.physical_start = 1, + 5 => descriptor.acknowledged_cursor = 1, + 6 => invalid.writer_epoch = 0, + 7 => invalid.chunk_type = crowdb_protocol::chunkdb::rpc::ChunkType::S3 as i32, + 8 => invalid + .cleanup_intents + .push(crowdb_protocol::chunkdb::rpc::StripCleanupIntent::default()), + 9 => invalid.strips.clone_from(&allocated.strips), + _ => invalid.state = ChunkState::Active as i32, + } + *allocator.chunk.lock().unwrap() = Some(invalid); + assert!(matches!( + cold.recover_tail(name, 10, &descriptor).await, + Err(crowdb_chunk_stream::StreamError::Corruption(_)) + )); + } +} + +#[tokio::test] +async fn finalized_empty_stream_tail_reopens_under_a_new_writer_with_a_fresh_chunk() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let make = || { + Arc::new( + ProductionStreamChunkStore::new( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + ) + .unwrap(), + ) + }; + let metadata = Arc::new(MemoryStreamStore::new(1024)); + let name = StreamName { high: 814, low: 100 }; + let stream = ChunkStream::create( + StreamRecord { + purpose: crowdb_chunk_stream::StreamPurpose::Wal, + stream_name: name, + metadata_group_id: 7, + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: None, + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), + }, + 9, + StreamConfig::default(), + metadata.clone(), + metadata.clone(), + make(), + ) + .await + .unwrap(); + stream.release_writer().await.unwrap(); + let old = { + let mut guard = allocator.chunk.lock().unwrap(); + let chunk = guard.as_mut().unwrap(); + chunk.state = ChunkState::Deleted as i32; + chunk.sealed_ts_ms = 123; + chunk.modify_ts += 2; + chunk.capacity = 0; + chunk.strips.clear(); + chunk.clone() + }; + let reopened = ChunkStream::open( + name, + 10, + StreamConfig::default(), + metadata.clone(), + metadata, + make(), + ) + .await + .unwrap(); + assert_eq!(reopened.tail(), 0); + assert_eq!(*allocator.chunk.lock().unwrap(), Some(old.clone())); + let appended = reopened + .append(&[Bytes::from_static(b"fresh-writer")]) + .await + .unwrap(); + assert_ne!(appended.chunk_id, old.id); + assert_eq!( + reopened.read_at(0, 12).await.unwrap(), + Bytes::from_static(b"fresh-writer") + ); + reopened.release_writer().await.unwrap(); +} diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/growth.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/growth.rs index a5d5c683d..7c9cb3b7a 100644 --- a/lib/crowdb-chunk-stream/tests/production_chunk_test/growth.rs +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/growth.rs @@ -3,6 +3,36 @@ use super::*; +#[tokio::test] +async fn growth_respects_bounded_initialization_admission() { + use super::initialization_window::TestInitializationWindow; + + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let window = Arc::new(TestInitializationWindow::new(disks.clone(), 4)); + let store = + ProductionStreamChunkStore::new(allocator, window.clone(), 30_000, ChunkReadPolicy::default()) + .unwrap(); + let name = StreamName { high: 197, low: 199 }; + let active = store + .allocate_mirrored(name, 9, crowdb_chunk_stream::StreamPurpose::Stream) + .await + .unwrap(); + let grown = store + .grow_mirrored(name, 9, active.chunk_id, 8 * 1024 * 1024) + .await + .unwrap() + .unwrap(); + assert_eq!(grown.capacity, 8 * 1024 * 1024); + assert_eq!(window.rejected(), 0); + assert_eq!(disks.fsyncs.load(Ordering::Acquire), 16); + let media = disks.bytes.lock().unwrap(); + assert_eq!(media.len(), 16); + assert!(media + .values() + .all(|bytes| bytes.len() == 1024 * 1024 && bytes.iter().all(|byte| *byte == 0))); +} + #[tokio::test] async fn a_small_waiting_append_uses_a_smaller_growth_window() { let allocator = Arc::new(Allocator::new()); @@ -56,13 +86,13 @@ async fn a_small_waiting_append_uses_a_smaller_growth_window() { .await .unwrap() .unwrap(); - assert_eq!(grown.capacity, 8 * 1024 * 1024); + assert_eq!(grown.capacity, 4 * 1024 * 1024); assert_eq!(grown.acknowledged_cursor, end); - assert_eq!(disks.fsyncs.load(Ordering::Acquire), 18); + assert_eq!(disks.fsyncs.load(Ordering::Acquire), 10); } #[tokio::test] -async fn growth_amortizes_allocation_and_initializes_every_replica_within_the_hard_capacity() { +async fn growth_honors_requested_capacity_and_initializes_every_replica_within_the_hard_limit() { const MIB: u64 = 1024 * 1024; let allocator = Arc::new(Allocator::new()); let disks = Arc::new(Disks::default()); @@ -82,7 +112,7 @@ async fn growth_amortizes_allocation_and_initializes_every_replica_within_the_ha .unwrap(); assert_eq!(active.capacity, MIB); let grown = store - .grow_mirrored(name, 9, active.chunk_id, MIB + 1) + .grow_mirrored(name, 9, active.chunk_id, 8 * MIB) .await .unwrap() .unwrap(); @@ -107,7 +137,7 @@ async fn growth_amortizes_allocation_and_initializes_every_replica_within_the_ha assert_eq!(disks.fsyncs.load(Ordering::Acquire), 16); assert_eq!( store - .grow_mirrored(name, 9, active.chunk_id, 9 * MIB) + .grow_mirrored(name, 9, active.chunk_id, 10 * MIB) .await .unwrap() .unwrap() diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/local_progress.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/local_progress.rs index 2205e14f1..d24254758 100644 --- a/lib/crowdb-chunk-stream/tests/production_chunk_test/local_progress.rs +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/local_progress.rs @@ -102,7 +102,7 @@ async fn grouped_progress_survives_checkpoint_query_renew_growth_and_mirror_repa store.renew_liveness(active.chunk_id, 9).await.unwrap(); let old_layout = allocator.chunk.lock().unwrap().clone(); let grown = store - .grow_mirrored(name, 9, active.chunk_id, active.capacity + 1) + .grow_mirrored(name, 9, active.chunk_id, 4 * 1024 * 1024) .await .unwrap() .unwrap(); diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/preparation.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/preparation.rs index 152b8b7ae..60f40d5c8 100644 --- a/lib/crowdb-chunk-stream/tests/production_chunk_test/preparation.rs +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/preparation.rs @@ -4,6 +4,29 @@ use super::*; use tokio::time::{timeout, Duration}; +#[tokio::test] +async fn a_small_cold_stream_does_not_initialize_megabytes_of_unused_spares() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let stream = test_stream(allocator.clone(), disks.clone()).await; + stream.append(&[Bytes::from_static(b"retained")]).await.unwrap(); + stream.release_writer().await.unwrap(); + assert!(allocator.successors.lock().unwrap().is_empty()); + let initialized: usize = disks + .writes + .lock() + .unwrap() + .iter() + .filter(|(_, _, _, bytes)| bytes.len() == 1024 * 1024) + .map(|(_, _, _, bytes)| bytes.len()) + .sum(); + assert_eq!(initialized, 2 * 1024 * 1024); + assert_eq!( + stream.read_at(0, 8).await.unwrap(), + Bytes::from_static(b"retained") + ); +} + #[tokio::test] async fn preparation_revision_does_not_abort_or_reinitialize_mirror_repair() { let allocator = Arc::new(Allocator::new()); @@ -81,7 +104,8 @@ async fn temporary_allocation_failure_does_not_poison_later_capacity() { allocator.pause_growth.store(true, Ordering::Release); allocator.fail_growth_once.store(true, Ordering::Release); let stream = test_stream(allocator.clone(), Arc::new(Disks::default())).await; - stream.append(&[Bytes::from_static(b"retained")]).await.unwrap(); + let retained = Bytes::from(vec![0x51; 512 * 1024]); + stream.append(std::slice::from_ref(&retained)).await.unwrap(); timeout(Duration::from_secs(1), allocator.growth_started.notified()) .await .unwrap(); @@ -91,7 +115,8 @@ async fn temporary_allocation_failure_does_not_poison_later_capacity() { .await .unwrap() .unwrap(); - assert_eq!(range.begin, 8); + assert_eq!(range.begin, retained.len() as u64); + assert_eq!(stream.read_at(0, retained.len()).await.unwrap(), retained); assert_eq!(stream.read_at(range.begin, data.len()).await.unwrap(), data); stream.close().await.unwrap(); } @@ -103,8 +128,9 @@ async fn uncertain_unused_candidate_is_reclaimed_and_replaced_without_reinitiali // The primary chunk uses disks 1/2, 11/12, ...; its first candidate // has distinct disks 1001/1002, 1011/1012, ... . Fail candidate growth. disks.fail_fsync_once.store(1011, Ordering::Release); - let stream = test_stream(allocator.clone(), disks.clone()).await; - stream.append(&[Bytes::from_static(b"retained")]).await.unwrap(); + let stream = test_stream_with_capacity(allocator.clone(), disks.clone(), 2 * 1024 * 1024).await; + let retained = Bytes::from(vec![0x51; 1536 * 1024]); + stream.append(std::slice::from_ref(&retained)).await.unwrap(); timeout(Duration::from_secs(1), async { while allocator.successors.lock().unwrap().len() < 2 { tokio::task::yield_now().await; @@ -133,13 +159,18 @@ async fn uncertain_unused_candidate_is_reclaimed_and_replaced_without_reinitiali ); assert_eq!(stream.metrics().failed, 0); assert_eq!(stream.metrics().rollovers, 0); - assert_eq!( - stream.read_at(0, 8).await.unwrap(), - Bytes::from_static(b"retained") - ); + assert_eq!(stream.read_at(0, retained.len()).await.unwrap(), retained); } async fn test_stream(allocator: Arc, disks: Arc) -> ChunkStream { + test_stream_with_capacity(allocator, disks, 8 * 1024 * 1024).await +} + +async fn test_stream_with_capacity( + allocator: Arc, + disks: Arc, + capacity: u64, +) -> ChunkStream { let metadata = Arc::new(MemoryStreamStore::new(256 * 1024 * 1024)); let chunks = Arc::new( ProductionStreamChunkStore::new(allocator, disks, 30_000, ChunkReadPolicy::default()).unwrap(), @@ -156,7 +187,10 @@ async fn test_stream(allocator: Arc, disks: Arc) -> ChunkStrea created_at_ms: StreamRecord::creation_time_ms(), }, 9, - StreamConfig::default(), + StreamConfig { + chunk_capacity_bytes: capacity, + ..StreamConfig::default() + }, metadata.clone(), metadata, chunks, @@ -170,10 +204,10 @@ async fn allocation_in_flight_does_not_block_writes_in_ready_capacity() { let allocator = Arc::new(Allocator::new()); allocator.pause_growth.store(true, Ordering::Release); let stream = test_stream(allocator.clone(), Arc::new(Disks::default())).await; - let payload = Bytes::from(vec![0x51; 4096]); + let retained = Bytes::from(vec![0x51; 512 * 1024]); timeout( Duration::from_secs(1), - stream.append(std::slice::from_ref(&payload)), + stream.append(std::slice::from_ref(&retained)), ) .await .unwrap() @@ -181,6 +215,7 @@ async fn allocation_in_flight_does_not_block_writes_in_ready_capacity() { timeout(Duration::from_secs(1), allocator.growth_started.notified()) .await .unwrap(); + let payload = Bytes::from(vec![0x52; 4096]); let range = timeout( Duration::from_secs(1), stream.append(std::slice::from_ref(&payload)), @@ -188,7 +223,7 @@ async fn allocation_in_flight_does_not_block_writes_in_ready_capacity() { .await .unwrap() .unwrap(); - assert_eq!(range.begin, 4096); + assert_eq!(range.begin, retained.len() as u64); allocator.growth_resume.notify_one(); stream.close().await.unwrap(); assert_eq!(stream.read_at(range.begin, payload.len()).await.unwrap(), payload); @@ -200,7 +235,8 @@ async fn only_a_write_reaching_uninitialized_capacity_waits() { let disks = Arc::new(Disks::default()); disks.pause_initialization.store(true, Ordering::Release); let stream = test_stream(allocator.clone(), disks.clone()).await; - stream.append(&[Bytes::from(vec![0x31; 4096])]).await.unwrap(); + let retained = Bytes::from(vec![0x31; 512 * 1024]); + stream.append(std::slice::from_ref(&retained)).await.unwrap(); timeout(Duration::from_secs(1), disks.initialization_started.notified()) .await .unwrap(); @@ -216,14 +252,14 @@ async fn only_a_write_reaching_uninitialized_capacity_waits() { let data = large.clone(); let mut pending = tokio::spawn(async move { writer.append(&[data]).await }); assert!(timeout(Duration::from_millis(20), &mut pending).await.is_err()); - assert_eq!(stream.tail(), 8192); + assert_eq!(stream.tail(), retained.len() as u64 + 4096); disks.initialization_resume.notify_one(); let range = timeout(Duration::from_secs(1), pending) .await .unwrap() .unwrap() .unwrap(); - assert_eq!(range.begin, 8192); + assert_eq!(range.begin, retained.len() as u64 + 4096); assert_eq!(stream.read_at(range.begin, large.len()).await.unwrap(), large); stream.close().await.unwrap(); } @@ -234,7 +270,10 @@ async fn close_drains_preparation_before_sealing() { let disks = Arc::new(Disks::default()); disks.pause_initialization.store(true, Ordering::Release); let stream = test_stream(allocator.clone(), disks.clone()).await; - stream.append(&[Bytes::from_static(b"retained")]).await.unwrap(); + stream + .append(&[Bytes::from(vec![0x51; 512 * 1024])]) + .await + .unwrap(); timeout(Duration::from_secs(1), disks.initialization_started.notified()) .await .unwrap(); diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/recovery.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/recovery.rs index d3bb7085e..5ad1e196d 100644 --- a/lib/crowdb-chunk-stream/tests/production_chunk_test/recovery.rs +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/recovery.rs @@ -86,3 +86,69 @@ async fn cold_adapter_uses_the_shortest_healthy_complete_prefix() { async fn cold_adapter_does_not_turn_a_corrupt_replica_into_an_empty_tail() { recover_case(false, true).await; } + +#[tokio::test] +async fn cold_adapter_recovers_an_empty_sealed_tail_after_its_strips_are_reclaimed() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let make = || { + ProductionStreamChunkStore::new( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + ) + .unwrap() + }; + let writer = make(); + let name = StreamName { high: 803, low: 93 }; + let mut active = writer + .allocate_mirrored(name, 9, crowdb_chunk_stream::StreamPurpose::Wal) + .await + .unwrap(); + active.logical_start = 42; + // DiskDB returns all strips when ChunkDB seals a physically empty chunk. + let sealed = { + let mut chunk = allocator.chunk.lock().unwrap(); + let chunk = chunk.as_mut().unwrap(); + chunk.state = ChunkState::Sealed as i32; + chunk.modify_ts += 1; + chunk.capacity = 0; + chunk.strips.clear(); + chunk.clone() + }; + drop(writer); + let cold = make(); + let memory = MemoryStreamStore::new(1024); + cold.bind_writer(name, 10, memory.claim_writer(name, 10).await.unwrap()); + let recovered = cold.recover_tail(name, 10, &active).await.unwrap(); + assert_eq!((recovered.physical_end, recovered.logical_end), (0, 42)); + assert!(recovered.extents.is_empty()); + for case in 0..8 { + let mut invalid = sealed.clone(); + let mut descriptor = active.clone(); + match case { + 0 => invalid.state = ChunkState::Active as i32, + 1 => invalid.capacity = 1, + 2 => invalid.sealed_length = 1, + 3 => invalid.acknowledged_cursor = 1, + 4 => descriptor.acknowledged_cursor = 1, + 5 => descriptor.physical_start = 1, + 6 => invalid.writer_epoch = 0, + _ => invalid.chunk_type = crowdb_protocol::chunkdb::rpc::ChunkType::S3 as i32, + } + *allocator.chunk.lock().unwrap() = Some(invalid); + assert!(matches!( + cold.recover_tail(name, 10, &descriptor).await, + Err(crowdb_chunk_stream::StreamError::Corruption(_)) + )); + } + let mut changed = sealed.clone(); + changed.modify_ts += 1; + *allocator.chunk.lock().unwrap() = Some(changed); + *allocator.stale_query_once.lock().unwrap() = Some(sealed); + assert!(matches!( + cold.recover_tail(name, 10, &active).await, + Err(crowdb_chunk_stream::StreamError::StaleWriter) + )); +} diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/recovery_drain.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/recovery_drain.rs new file mode 100644 index 000000000..e76a23264 --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/recovery_drain.rs @@ -0,0 +1,145 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; + +#[tokio::test(start_paused = true)] +async fn recovery_drains_a_large_layout_with_bounded_parallelism_across_strips() { + use std::time::Duration; + + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let make = || { + ProductionStreamChunkStore::new( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + ) + .unwrap() + }; + let writer = make(); + let name = StreamName::generate(); + let active = writer + .allocate_mirrored(name, 9, crowdb_chunk_stream::StreamPurpose::Wal) + .await + .unwrap(); + writer + .grow_mirrored(name, 9, active.chunk_id, 32 * 1024 * 1024) + .await + .unwrap(); + let mut encoded = TestGroupedStrip::default(); + let end = encoded + .append(&writer, name, 9, active.chunk_id, b"retained") + .await; + drop(writer); + let cold = make(); + let memory = MemoryStreamStore::new(1024); + cold.bind_writer(name, 10, memory.claim_writer(name, 10).await.unwrap()); + disks.drain_delay_ms.store(20, Ordering::Release); + let recovered = tokio::time::timeout(Duration::from_millis(400), cold.recover_tail(name, 10, &active)) + .await + .expect("all allocated strips must drain within the original recovery budget") + .unwrap(); + assert_eq!((recovered.physical_end, recovered.logical_end), (end, 8)); + assert_eq!(recovered.extents.len(), 1); + assert_eq!(disks.completed_drains.load(Ordering::Acquire), 64); + assert!((4..=8).contains(&disks.max_active_drains.load(Ordering::Acquire))); + assert_eq!(disks.active_drains.load(Ordering::Acquire), 0); +} + +#[tokio::test] +async fn a_failed_recovery_drain_requests_layout_repair_before_tail_observation() { + for lost_reply in [false, true] { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let make = || { + ProductionStreamChunkStore::new( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + ) + .unwrap() + }; + let writer = make(); + let name = StreamName { high: 811, low: 97 }; + let active = writer + .allocate_mirrored(name, 9, crowdb_chunk_stream::StreamPurpose::Wal) + .await + .unwrap(); + let mut encoded = TestGroupedStrip::default(); + let end = encoded + .append(&writer, name, 9, active.chunk_id, b"retained") + .await; + drop(writer); + let cold = make(); + let memory = MemoryStreamStore::new(1024); + cold.bind_writer(name, 10, memory.claim_writer(name, 10).await.unwrap()); + disks.fail_fsync_once.store(2, Ordering::Release); + allocator + .fail_replacement_response_once + .store(lost_reply, Ordering::Release); + assert!(cold.recover_tail(name, 10, &active).await.is_err()); + assert_eq!(disks.completed_drains.load(Ordering::Acquire), 2); + assert_eq!(allocator.replacement_calls.load(Ordering::Acquire), 1); + let pending = allocator.chunk.lock().unwrap().clone().unwrap(); + assert_eq!(pending.state, ChunkState::Active as i32); + let Some(Strip::MirrorStrip(mirror)) = &pending.strips[0].strip else { + panic!("mirror") + }; + assert_eq!(pending.strips[0].unavailable_segments, vec![mirror.segments[1]]); + // Even if a transport recovers, the published unavailable layout still + // needs its durable repair before it can authorize recovery. + assert!(cold.recover_tail(name, 10, &active).await.is_err()); + assert_eq!(allocator.replacement_calls.load(Ordering::Acquire), 1); + // Model the existing repair executor copying the drained surviving mirror + // into a replacement and publishing its new allocation identity. + { + let mut bytes = disks.bytes.lock().unwrap(); + let retained = bytes.get(&1).unwrap().clone(); + bytes.insert(101, retained); + } + { + let mut chunk = allocator.chunk.lock().unwrap(); + let chunk = chunk.as_mut().unwrap(); + let strip = &mut chunk.strips[0]; + let Some(Strip::MirrorStrip(mirror)) = &mut strip.strip else { + panic!("mirror") + }; + mirror.segments[1].disk_id.as_mut().unwrap().high = 101; + mirror.segments[1].allocation_ts += 1; + strip.unavailable_segments.clear(); + chunk.modify_ts += 1; + } + let recovered = cold.recover_tail(name, 10, &active).await.unwrap(); + assert_eq!((recovered.physical_end, recovered.logical_end), (end, 8)); + assert_eq!(recovered.extents.len(), 1); + } +} + +#[tokio::test] +async fn failed_drains_on_every_mirror_cannot_admit_repair_or_a_tail() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let store = ProductionStreamChunkStore::new( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + ) + .unwrap(); + let memory = MemoryStreamStore::new(1024); + let name = StreamName::generate(); + store.bind_writer(name, 10, memory.claim_writer(name, 10).await.unwrap()); + let active = store + .allocate_mirrored(name, 10, crowdb_chunk_stream::StreamPurpose::Wal) + .await + .unwrap(); + let before = allocator.chunk.lock().unwrap().clone(); + disks.fail_all_drains.store(true, Ordering::Release); + assert!(store.recover_tail(name, 10, &active).await.is_err()); + assert_eq!(disks.completed_drains.load(Ordering::Acquire), 2); + assert_eq!(allocator.replacement_calls.load(Ordering::Acquire), 0); + assert_eq!(*allocator.chunk.lock().unwrap(), before); +} diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/unwritten.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/unwritten.rs index 24a2b3127..1fd1584ac 100644 --- a/lib/crowdb-chunk-stream/tests/production_chunk_test/unwritten.rs +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/unwritten.rs @@ -180,7 +180,10 @@ async fn cross_strip_tail_header_survives_later_growth_and_the_next_append() { created_at_ms: StreamRecord::creation_time_ms(), }, 9, - StreamConfig::default(), + StreamConfig { + chunk_capacity_bytes: 2 * 1024 * 1024, + ..StreamConfig::default() + }, metadata.clone(), metadata.clone(), chunks.clone(), @@ -192,11 +195,11 @@ async fn cross_strip_tail_header_survives_later_growth_and_the_next_append() { let chunk_id = range.chunk_id.unwrap(); let complete_end = 1024 * 1024 - 7; assert_eq!(chunks.durable_cursor(chunk_id, 9).await.unwrap().offset, 0); - assert_eq!(chunks.current_capacity(chunk_id).await.unwrap(), 8 * 1024 * 1024); + assert_eq!(chunks.current_capacity(chunk_id).await.unwrap(), 2 * 1024 * 1024); assert_eq!(observed_end(&allocator, &disks, name).await, complete_end); let publications = metadata.metadata_publish_count(); chunks - .grow_mirrored(name, 9, chunk_id, 9 * 1024 * 1024) + .grow_mirrored(name, 9, chunk_id, 3 * 1024 * 1024) .await .unwrap() .unwrap(); @@ -281,7 +284,7 @@ async fn uncertain_growth_initialization_cannot_be_retried_or_written() { let end = strip.append(&store, name, 9, active.chunk_id, b"retained").await; disks.fail_fsync_once.store(11, Ordering::Release); assert!(store - .grow_mirrored(name, 9, active.chunk_id, active.capacity + 1) + .grow_mirrored(name, 9, active.chunk_id, 4 * 1024 * 1024) .await .is_err()); // All three added strips complete their submitted work, including the healthy @@ -289,7 +292,7 @@ async fn uncertain_growth_initialization_cannot_be_retried_or_written() { assert_eq!(disks.fsyncs.load(Ordering::Acquire), 9); let submissions = disks.writes.lock().unwrap().len(); assert!(store - .grow_mirrored(name, 9, active.chunk_id, active.capacity + 1) + .grow_mirrored(name, 9, active.chunk_id, 4 * 1024 * 1024) .await .is_err()); assert_eq!(disks.writes.lock().unwrap().len(), submissions); diff --git a/lib/crowdb-chunkdb-client/src/client.rs b/lib/crowdb-chunkdb-client/src/client.rs index 49ad7bd7f..77232e373 100644 --- a/lib/crowdb-chunkdb-client/src/client.rs +++ b/lib/crowdb-chunkdb-client/src/client.rs @@ -212,6 +212,10 @@ impl ChunkdbClient { async fn endpoints_for_chunk(&self, chunk_id: Option<&ChunkId>) -> Result> { self.refresh_endpoints_if_due().await; + self.cached_endpoints_for_chunk(chunk_id).await + } + + async fn cached_endpoints_for_chunk(&self, chunk_id: Option<&ChunkId>) -> Result> { if let Some(id) = chunk_id { let binding = &self.range_binding; let owner = binding diff --git a/lib/crowdb-chunkdb-client/src/readiness.rs b/lib/crowdb-chunkdb-client/src/readiness.rs index d5d9fa913..9d9b79aad 100644 --- a/lib/crowdb-chunkdb-client/src/readiness.rs +++ b/lib/crowdb-chunkdb-client/src/readiness.rs @@ -31,7 +31,14 @@ impl ChunkdbClient { high: u64::from(purpose) << 56, low: 0, }; - match self.query_chunk(QueryChunkRequest { chunk_id: Some(id) }).await { + let reply = self + .with_probe_retry(&id, |transport, endpoint| async move { + transport + .send_query_chunk(&endpoint, &QueryChunkRequest { chunk_id: Some(id) }) + .await + }) + .await; + match reply { Ok(reply) if reply.chunk.as_ref().is_some_and(|chunk| chunk.id == Some(id)) => Ok(()), Err(ChunkdbClientError::NotFound(_)) => Ok(()), Ok(_) => Err(ChunkdbClientError::Rpc( diff --git a/lib/crowdb-chunkdb-client/src/retry.rs b/lib/crowdb-chunkdb-client/src/retry.rs index b66dde139..eab2b26eb 100644 --- a/lib/crowdb-chunkdb-client/src/retry.rs +++ b/lib/crowdb-chunkdb-client/src/retry.rs @@ -7,6 +7,11 @@ use crowdb_protocol::chunkdb::rpc::{AllocateChunkRequest, AllocateChunkResponse} use crowdb_protocol::common::ChunkId; use std::{sync::Arc, time::Duration}; +enum RouteRefresh { + Periodic, + AfterFailure, +} + impl ChunkdbClient { pub(super) async fn with_rpc_retry(&self, chunk_id: Option<&ChunkId>, op: F) -> Result where @@ -14,7 +19,7 @@ impl ChunkdbClient { Fut: std::future::Future>, { self.rpc_transport - .with_request_identity(self.retry_rpc(chunk_id, false, op)) + .with_request_identity(self.retry_rpc(chunk_id, false, RouteRefresh::Periodic, op)) .await } @@ -24,12 +29,30 @@ impl ChunkdbClient { Fut: std::future::Future>, { self.rpc_transport - .with_request_identity(self.retry_rpc(chunk_id, true, op)) + .with_request_identity(self.retry_rpc(chunk_id, true, RouteRefresh::Periodic, op)) + .await + } + + /// A real readiness point read must not enumerate authority before using + /// an initialized route. Cold discovery and failed-route recovery remain. + pub(crate) async fn with_probe_retry(&self, chunk_id: &ChunkId, op: F) -> Result + where + F: Fn(Arc, String) -> Fut, + Fut: std::future::Future>, + { + self.rpc_transport + .with_request_identity(self.retry_rpc(Some(chunk_id), true, RouteRefresh::AfterFailure, op)) .await } /// Execute a crowdb-rpc call with retry on transient errors. - async fn retry_rpc(&self, chunk_id: Option<&ChunkId>, read_only: bool, op: F) -> Result + async fn retry_rpc( + &self, + chunk_id: Option<&ChunkId>, + read_only: bool, + refresh: RouteRefresh, + op: F, + ) -> Result where F: Fn(Arc, String) -> Fut, Fut: std::future::Future>, @@ -39,7 +62,10 @@ impl ChunkdbClient { let mut attempts = 0u32; let mut backoff = self.retry.initial_backoff; loop { - let endpoints = self.endpoints_for_chunk(chunk_id).await?; + let endpoints = match refresh { + RouteRefresh::Periodic => self.endpoints_for_chunk(chunk_id).await?, + RouteRefresh::AfterFailure => self.cached_endpoints_for_chunk(chunk_id).await?, + }; let mut last_error = None; for endpoint in endpoints { match tokio::time::timeout_at(deadline, op(Arc::clone(&transport), endpoint)) @@ -123,8 +149,13 @@ impl ChunkdbClient { }) { Ok(response) => return Ok(response), Err( - error @ (ChunkdbClientError::NotMyRange(_) | ChunkdbClientError::ConnectFailed(_)), + error @ (ChunkdbClientError::NotMyRange(_) + | ChunkdbClientError::ConnectFailed(_) + | ChunkdbClientError::Unavailable(_)), ) => { + // The server reports KV quorum loss as Unavailable + // only after allocation has rolled back its physical + // strips, so retrying starts a fresh safe allocation. safe_retry = Some(error); } Err(error) => return Err(error), diff --git a/lib/crowdb-console-shared/src/clients/http.rs b/lib/crowdb-console-shared/src/clients/http.rs index 717b76810..afdf1cc2c 100644 --- a/lib/crowdb-console-shared/src/clients/http.rs +++ b/lib/crowdb-console-shared/src/clients/http.rs @@ -87,12 +87,19 @@ impl ServerClient { self.get_json("/health").await } - /// `GET /topology`. + /// `GET /topology`. Resolve wildcard store listeners against the + /// reporting management origin before callers cache them. Explicit peer + /// endpoints retain their advertised addresses. /// /// # Errors /// Returns `Error::UpstreamRpc` for transport / decode failures. pub async fn topology(&self) -> Result> { - let resp: TopologyResponse = self.get_json("/topology").await?; + let mut resp: TopologyResponse = self.get_json("/topology").await?; + for store in &mut resp.stores { + if let Some(endpoint) = &store.listen_addr { + store.listen_addr = Some(self.resolve_rpc_endpoint(endpoint)?); + } + } Ok(resp.stores) } diff --git a/lib/crowdb-console-shared/src/deployment/admission.rs b/lib/crowdb-console-shared/src/deployment/admission.rs index 270b7e689..a2dc9cb8f 100644 --- a/lib/crowdb-console-shared/src/deployment/admission.rs +++ b/lib/crowdb-console-shared/src/deployment/admission.rs @@ -15,11 +15,16 @@ use crate::config::NodeEntry; use crate::error::{Error, Result}; use crate::ssh::{Session, SshCreds}; +mod process_observation; +pub use process_observation::RemoteProcessObservation; + #[derive(Clone, Debug, Serialize, Deserialize)] pub struct SshNodeIdentity { pub discovery_id: String, pub public_key: String, pub host_key: String, + #[serde(default = "super::registry::default_web_port")] + pub web_port: u16, } /// # Errors diff --git a/lib/crowdb-console-shared/src/deployment/admission/process_observation.rs b/lib/crowdb-console-shared/src/deployment/admission/process_observation.rs new file mode 100644 index 000000000..00491aebd --- /dev/null +++ b/lib/crowdb-console-shared/src/deployment/admission/process_observation.rs @@ -0,0 +1,83 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! A single sampler owns its authenticated, read-only monitor connection. + +use crate::{ + config::NodeEntry, + error::{Error, Result}, + ssh::Session, +}; +use crowdb_protocol::mgmt::node::{NodeBinding, NodeControl}; +use serde_json::Value; +use std::time::Duration; +use tokio::time::Instant; + +/// Bound process observations reuse SSH without sharing sessions with mutations. +/// Dropping a read future discards its connection; the next cadence reconnects. +pub struct RemoteProcessObservation { + node: NodeEntry, + binding: NodeBinding, + connection: Option<(Session, Instant)>, +} + +impl RemoteProcessObservation { + /// Construct a reader for one accepted node using its provisioned local key. + /// + /// # Errors + /// Rejects password fallback or a missing explicit key. + pub fn new(node: NodeEntry, binding: NodeBinding) -> Result { + if node.ssh_password.is_some() || node.ssh_key.is_none() { + return Err(Error::Config( + "process observations require an explicit node key".into(), + )); + } + Ok(Self { + node, + binding, + connection: None, + }) + } + + /// Observe the same node and bootstrap, with no transport retry on failure. + /// + /// # Errors + /// Propagates authentication, command and identity failures. + pub async fn read(&mut self) -> Result { + // Keep no stored handle across an await: timeout cancellation must not + // retain a connection with an unfinished command or an unread reply. + let (mut session, connected) = match self.connection.take() { + Some((session, connected)) if connected.elapsed() < Duration::from_secs(60) => { + (session, connected) + } + previous => { + if let Some((session, _)) = previous { + session.close().await; + } + (super::connect(&self.node).await?, Instant::now()) + } + }; + let command = NodeControl::Processes { + cluster_id: self.binding.bootstrap.cluster_id.clone(), + node_id: self.node.id, + }; + let result = super::control(&mut session, &command).await.and_then(|value| { + if value["node_id"] != self.node.id + || value["cluster_id"] != self.binding.bootstrap.cluster_id + || value["bootstrap"] + != serde_json::to_value(&self.binding.bootstrap).map_err(super::config_error)? + { + return Err(Error::Config( + "process observation identity differs from accepted node".into(), + )); + } + Ok(value) + }); + if result.is_ok() { + self.connection = Some((session, connected)); + } else { + session.close().await; + } + result + } +} diff --git a/lib/crowdb-console-shared/src/deployment/preparation/record.rs b/lib/crowdb-console-shared/src/deployment/preparation/record.rs index b60d58586..5102cacda 100644 --- a/lib/crowdb-console-shared/src/deployment/preparation/record.rs +++ b/lib/crowdb-console-shared/src/deployment/preparation/record.rs @@ -69,9 +69,9 @@ impl DefaultPreparation { fn with_operation(bootstrap: &PreparedBootstrap, operation_id: String) -> Result { let members = bootstrap.intent.members(); - if !matches!(members.len(), 1 | 3) { + if !matches!(members.len(), 1 | 3 | 5) { return Err(Error::Config( - "default preparation requires one or three admitted voters".into(), + "default preparation requires one, three or five admitted voters".into(), )); } let config = bootstrap.intent.to_config(); diff --git a/lib/crowdb-console-shared/src/deployment/registry.rs b/lib/crowdb-console-shared/src/deployment/registry.rs index 87b0cd3b7..1ebe3b1db 100644 --- a/lib/crowdb-console-shared/src/deployment/registry.rs +++ b/lib/crowdb-console-shared/src/deployment/registry.rs @@ -16,6 +16,8 @@ pub struct NodeRecord { pub physical_host_id: String, pub rack_id: u64, pub host: String, + #[serde(default = "default_web_port", skip_serializing_if = "IS_DEFAULT_WEB_PORT")] + pub web_port: u16, pub ssh_port: u16, pub ssh_user: String, pub operation_id: String, @@ -27,6 +29,13 @@ pub struct NodeRecord { pub cancelled: bool, } +pub(super) const fn default_web_port() -> u16 { + // Retained packaged-node records predate explicit Web listener publication. + 9090 +} + +const IS_DEFAULT_WEB_PORT: fn(&u16) -> bool = |port| *port == default_web_port(); + #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] pub struct NodeRegistry { pub cluster_id: String, diff --git a/lib/crowdb-console-shared/src/ops/cluster/readiness.rs b/lib/crowdb-console-shared/src/ops/cluster/readiness.rs index 767ccd3de..8c9ba0571 100644 --- a/lib/crowdb-console-shared/src/ops/cluster/readiness.rs +++ b/lib/crowdb-console-shared/src/ops/cluster/readiness.rs @@ -185,6 +185,7 @@ fn required(observation: &ClusterObservation, kind: DependencyKind) -> bool { return true; }; match kind { + DependencyKind::Metadata => policy.setup.requires_default_preparation(), DependencyKind::Storage => !matches!(policy.setup.storage, SetupStorage::Diskless), DependencyKind::Protocol => policy .setup @@ -197,7 +198,7 @@ fn required(observation: &ClusterObservation, kind: DependencyKind) -> bool { fn invalid(observation: &ClusterObservation) -> bool { observation.version != 1 - || !matches!(observation.initial_voters, 1 | 3) + || !matches!(observation.initial_voters, 1 | 3 | 5) || (observation.bound && observation.cluster_id.is_none()) || observation.cluster_id.as_ref().is_some_and(|identity| { uuid::Uuid::parse_str(identity).map_or(true, |identity| identity.is_nil()) diff --git a/lib/crowdb-console-shared/src/ops/cluster/readiness/collect.rs b/lib/crowdb-console-shared/src/ops/cluster/readiness/collect.rs index 3105a2587..d96d3477e 100644 --- a/lib/crowdb-console-shared/src/ops/cluster/readiness/collect.rs +++ b/lib/crowdb-console-shared/src/ops/cluster/readiness/collect.rs @@ -129,20 +129,36 @@ async fn gather( let (nodes, intents, metadata, storage) = tokio::join!( tokio::time::timeout_at(deadline, registry::read(ctx.kv())), tokio::time::timeout_at(deadline, services::list(ctx.kv())), - tokio::time::timeout_at(deadline, storage::metadata(ctx, bootstrap)), - tokio::time::timeout_at(deadline, storage::read(ctx, bootstrap)), + async { + if local.policy.setup.requires_default_preparation() { + Some(tokio::time::timeout_at(deadline, storage::metadata(ctx, bootstrap)).await) + } else { + None + } + }, + async { + if local.policy.setup.requires_default_preparation() { + tokio::time::timeout_at(deadline, storage::read(ctx, bootstrap)) + .await + .ok() + .and_then(Result::ok) + } else { + None + } + }, ); - observation - .dependencies - .push(metadata.ok().and_then(Result::ok).unwrap_or_else(|| { - fact( - "cluster/metadata", - DependencyKind::Metadata, - DependencyState::Unknown, - "metadata-unavailable", - ) - })); - let storage = storage.ok().and_then(Result::ok); + if let Some(metadata) = metadata { + observation + .dependencies + .push(metadata.ok().and_then(Result::ok).unwrap_or_else(|| { + fact( + "cluster/metadata", + DependencyKind::Metadata, + DependencyState::Unknown, + "metadata-unavailable", + ) + })); + } let Ok(Ok(Some((registry, _)))) = nodes else { append_storage(observation, storage.as_ref(), &std::collections::BTreeMap::new()); return; @@ -173,6 +189,13 @@ fn append_storage( storage: Option<&storage::StorageSnapshot>, nodes: &std::collections::BTreeMap, ) { + if observation + .policy + .as_ref() + .is_some_and(|policy| !policy.setup.requires_default_preparation()) + { + return; + } observation.dependencies.extend(storage.map_or_else( || { vec![fact( diff --git a/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/members.rs b/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/members.rs index e94cc2721..223f31deb 100644 --- a/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/members.rs +++ b/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/members.rs @@ -67,12 +67,13 @@ pub(super) async fn read( discovery_id: node.discovery_id.clone(), node_id: Some(node.node_id), management_endpoints: vec![format!( - "http://{}:9090", + "http://{}:{}", if host.contains(':') { format!("[{host}]") } else { host.into() - } + }, + node.web_port, )], }); let now = now_ms(); @@ -109,7 +110,10 @@ async fn member(client: &reqwest::Client, node: ®istry::NodeRecord) -> Option host.into() }; let mut reply = client - .get(format!("http://{host}:9090/api/cluster/node-observation")) + .get(format!( + "http://{host}:{}/api/cluster/node-observation", + node.web_port + )) .send() .await .ok()? diff --git a/lib/crowdb-console-shared/tests/cluster_readiness_test.rs b/lib/crowdb-console-shared/tests/cluster_readiness_test.rs index a29b9fc53..3bab3a90e 100644 --- a/lib/crowdb-console-shared/tests/cluster_readiness_test.rs +++ b/lib/crowdb-console-shared/tests/cluster_readiness_test.rs @@ -269,12 +269,29 @@ fn diskless_kv_only_policy_requires_its_selected_services_without_inventing_prot } policy.validate().unwrap(); input.policy = Some(policy); - input - .dependencies - .retain(|fact| !matches!(fact.kind, DependencyKind::Storage | DependencyKind::Protocol)); + input.dependencies.retain(|fact| { + !matches!( + fact.kind, + DependencyKind::Metadata | DependencyKind::Storage | DependencyKind::Protocol + ) + }); assert_eq!(reduce(input.clone(), 1000).phase, ClusterPhase::Ready); let mut remote_protocol = fact(DependencyKind::Protocol); remote_protocol.state = DependencyState::Unknown; input.dependencies.push(remote_protocol); assert_eq!(reduce(input, 1000).phase, ClusterPhase::Degraded); } + +#[test] +fn five_voter_readiness_preserves_required_evidence_and_rejects_other_counts() { + let mut input = observation(); + input.initial_voters = 5; + assert_eq!(reduce(input.clone(), 1000).phase, ClusterPhase::Ready); + input.dependencies[0].state = DependencyState::Unknown; + assert_eq!(reduce(input, 1000).phase, ClusterPhase::Unavailable); + for count in [0, 2, 4, 6, 7] { + let mut input = observation(); + input.initial_voters = count; + assert_eq!(reduce(input, 1000).phase, ClusterPhase::Failed); + } +} diff --git a/lib/crowdb-console-shared/tests/cluster_storage_observation_test.rs b/lib/crowdb-console-shared/tests/cluster_storage_observation_test.rs index 5159309ff..2bf45d7eb 100644 --- a/lib/crowdb-console-shared/tests/cluster_storage_observation_test.rs +++ b/lib/crowdb-console-shared/tests/cluster_storage_observation_test.rs @@ -280,6 +280,7 @@ async fn diskless_member_does_not_require_an_unused_storage_reservation() { &NodeRegistry { cluster_id: authority.bootstrap.identity.cluster_id.clone(), nodes: vec![NodeRecord { + web_port: 9090, discovery_id: node.discovery_id.clone(), node_id: 2, rack_id: 4, @@ -312,3 +313,106 @@ async fn diskless_member_does_not_require_an_unused_storage_reservation() { report.observation.dependencies ); } + +#[tokio::test] +async fn kv_only_collection_uses_confirmed_web_ports_without_preparing_storage() { + use crowdb_console_shared::deployment::registry::{self, NodeRecord, NodeRegistry}; + use crowdb_protocol::mgmt::{ + cluster::{ClusterPhase, DependencyKind}, + node::{ + observation::NodeServiceObservation, + setup::{SetupServiceKind, SetupStorage}, + }, + }; + let authority = TestPreparationAuthority::start(&[2]).await; + let mut node = local(&authority); + node.policy.setup.storage = SetupStorage::Diskless; + for service in &mut node.policy.setup.services { + service.enabled = service.kind == SetupServiceKind::PaxosKv; + } + node.observed_at_unix_ms = u64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(), + ) + .unwrap(); + node.services = vec![NodeServiceObservation { + service_id: "kv".into(), + kind: "kv".into(), + operation_id: None, + state: DependencyState::Ready, + detail: "current supervised KV".into(), + observed_at_unix_ms: node.observed_at_unix_ms, + max_age_ms: 3000, + reason_code: None, + }]; + let mut remote = node.clone(); + remote.discovery_id = uuid::Uuid::new_v4().to_string(); + remote.binding.as_mut().unwrap().node_id = 7; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let response = remote.clone(); + let task = tokio::spawn(async move { + axum::serve( + listener, + axum::Router::new().route( + "/api/cluster/node-observation", + axum::routing::get(move || { + let value = response.clone(); + async move { axum::Json(value) } + }), + ), + ) + .await + .unwrap(); + }); + let records = [ + (2, node.discovery_id.clone(), 9090), + (7, remote.discovery_id.clone(), port), + ] + .into_iter() + .map(|(node_id, discovery_id, web_port)| NodeRecord { + node_id, + discovery_id, + web_port, + rack_id: 4, + physical_host_id: "owned".into(), + host: "127.0.0.1".into(), + ssh_port: 2222, + ssh_user: "crowdb".into(), + operation_id: uuid::Uuid::new_v4().to_string(), + confirmed: true, + host_key: None, + cancelled: false, + }) + .collect(); + registry::publish( + authority.ctx.kv(), + &NodeRegistry { + cluster_id: authority.bootstrap.identity.cluster_id.clone(), + nodes: records, + }, + ) + .await + .unwrap(); + let report = collect(&authority.ctx, node).await.unwrap(); + task.abort(); + assert_eq!(report.phase, ClusterPhase::Ready, "{:?}", report.reasons); + assert_eq!(report.observation.members.len(), 2); + assert!(report + .observation + .members + .iter() + .any(|member| member.node_id == Some(7) + && member.management_endpoints == [format!("http://127.0.0.1:{port}")])); + assert!(!report.observation.dependencies.iter().any(|fact| matches!( + fact.kind, + DependencyKind::Metadata | DependencyKind::Storage | DependencyKind::Protocol + ))); + assert!(preparation::read(&authority.ctx, &authority.bootstrap.identity) + .await + .unwrap() + .is_none()); + assert!(authority.ctx.sysmd().get_group(0, 1).await.unwrap().is_none()); +} diff --git a/lib/crowdb-console-shared/tests/cluster_supplied_storage_observation_test.rs b/lib/crowdb-console-shared/tests/cluster_supplied_storage_observation_test.rs index 276b5ee80..837f72c94 100644 --- a/lib/crowdb-console-shared/tests/cluster_supplied_storage_observation_test.rs +++ b/lib/crowdb-console-shared/tests/cluster_supplied_storage_observation_test.rs @@ -61,6 +61,7 @@ async fn setup() -> (TestPreparationAuthority, NodeRuntimeObservation) { &NodeRegistry { cluster_id: authority.bootstrap.identity.cluster_id.clone(), nodes: vec![NodeRecord { + web_port: 9090, discovery_id: discovery_id.clone(), node_id: 2, physical_host_id: "owned-host".into(), diff --git a/lib/crowdb-console-shared/tests/common/member_evidence.rs b/lib/crowdb-console-shared/tests/common/member_evidence.rs index ed5221eb5..82c60d442 100644 --- a/lib/crowdb-console-shared/tests/common/member_evidence.rs +++ b/lib/crowdb-console-shared/tests/common/member_evidence.rs @@ -34,6 +34,7 @@ impl TestMemberEvidence { configuration_digest: "accepted-digest".into(), }; let expected = NodeRecord { + web_port: 9090, discovery_id: uuid::Uuid::new_v4().to_string(), node_id: 1, physical_host_id: "test-host".into(), diff --git a/lib/crowdb-console-shared/tests/common/ssh_server.rs b/lib/crowdb-console-shared/tests/common/ssh_server.rs index 276f2dbb3..03d5b108d 100644 --- a/lib/crowdb-console-shared/tests/common/ssh_server.rs +++ b/lib/crowdb-console-shared/tests/common/ssh_server.rs @@ -1,7 +1,10 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use std::sync::Arc; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; use async_trait::async_trait; use russh::keys::key::{KeyPair, PublicKey}; @@ -12,12 +15,14 @@ use russh::{Channel, ChannelId, CryptoVec}; #[allow(dead_code)] pub enum TestReply { Stdout(Vec), + DelayedStdout(Vec, std::time::Duration), Failure(String), } pub struct TestSshServer { pub port: u16, server: tokio::task::JoinHandle<()>, + connections: Arc, } impl Drop for TestSshServer { @@ -28,6 +33,9 @@ impl Drop for TestSshServer { #[allow(dead_code)] impl TestSshServer { + pub fn connections(&self) -> usize { + self.connections.load(Ordering::Relaxed) + } pub async fn start(key: PublicKey) -> Self { Self::start_with_reply(key, None).await } @@ -51,12 +59,15 @@ impl TestSshServer { keys: vec![host], ..Default::default() }); + let connections = Arc::new(AtomicUsize::new(0)); + let accepted_count = Arc::clone(&connections); let server = tokio::spawn(async move { let mut sessions = tokio::task::JoinSet::new(); loop { tokio::select! { accepted = listener.accept() => { let (stream, _) = accepted.unwrap(); + accepted_count.fetch_add(1, Ordering::Relaxed); let config = config.clone(); let handler = TestHandler { key: key.clone(), reply: reply.clone() }; sessions.spawn(async move { @@ -69,7 +80,11 @@ impl TestSshServer { } } }); - Self { port, server } + Self { + port, + server, + connections, + } } } @@ -109,6 +124,11 @@ impl Handler for TestHandler { session.data(channel, CryptoVec::from_slice(bytes)); session.exit_status_request(channel, 0); } + TestReply::DelayedStdout(bytes, delay) => { + tokio::time::sleep(*delay).await; + session.data(channel, CryptoVec::from_slice(bytes)); + session.exit_status_request(channel, 0); + } TestReply::Failure(error) => { session.extended_data(channel, 1, CryptoVec::from_slice(error.as_bytes())); session.exit_status_request(channel, 1); diff --git a/lib/crowdb-console-shared/tests/default_preparation_test.rs b/lib/crowdb-console-shared/tests/default_preparation_test.rs index 3e1851b2d..abe1b0090 100644 --- a/lib/crowdb-console-shared/tests/default_preparation_test.rs +++ b/lib/crowdb-console-shared/tests/default_preparation_test.rs @@ -362,3 +362,50 @@ async fn explicit_retirement_is_idempotent_and_prevents_automatic_recreation() { ); assert_eq!(snapshot(&authority).await.unwrap(), retired); } + +#[tokio::test] +async fn five_voter_preparation_retains_every_admitted_identity_and_fixed_mapping() { + let members = [2, 7, 9, 12, 15]; + let authority = TestPreparationAuthority::start(&members).await; + let winner = preparation::reserve(&authority.ctx, &authority.bootstrap.identity) + .await + .unwrap(); + assert_eq!( + winner + .metadata + .replicas + .iter() + .map(|r| (r.node_id, r.replica_id)) + .collect::>(), + vec![(2, 1), (7, 2), (9, 3), (12, 4), (15, 5)] + ); + assert_eq!( + winner + .storage + .iter() + .map(|b| (b.node_id, b.disk_group_id, b.rack_id, b.store_id, b.group_id)) + .collect::>(), + vec![ + (2, 1, 4, 0, 1), + (7, 2, 4, 0, 1), + (9, 3, 4, 0, 1), + (12, 4, 4, 0, 1), + (15, 5, 4, 0, 1) + ] + ); + assert_eq!(winner.chunk_kv.node_id, 2); + let before = snapshot(&authority).await.unwrap(); + let recovered = preparation::read(&authority.ctx, &authority.bootstrap.identity) + .await + .unwrap() + .unwrap(); + assert_eq!(recovered.0, winner); + assert_eq!(recovered.1, before.1); + assert_eq!( + preparation::reserve(&authority.ctx, &authority.bootstrap.identity) + .await + .unwrap(), + winner + ); + assert_eq!(snapshot(&authority).await.unwrap(), before); +} diff --git a/lib/crowdb-console-shared/tests/member_evidence_test.rs b/lib/crowdb-console-shared/tests/member_evidence_test.rs index 0dc09521c..88cd89882 100644 --- a/lib/crowdb-console-shared/tests/member_evidence_test.rs +++ b/lib/crowdb-console-shared/tests/member_evidence_test.rs @@ -172,3 +172,24 @@ fn duplicate_service_or_protocol_identity_is_a_conflict() { assert_eq!(fixture.project()[0].state, DependencyState::Failed); } } + +#[test] +fn default_web_port_preserves_legacy_records_and_offset_ports_round_trip() { + let mut record = TestMemberEvidence::new().expected; + let legacy = serde_json::to_value(&record).unwrap(); + assert!(legacy.get("web_port").is_none()); + assert_eq!( + serde_json::from_value::(legacy.clone()) + .unwrap(), + record + ); + record.web_port = 9092; + let shifted = serde_json::to_value(&record).unwrap(); + assert_eq!(shifted["web_port"], 9092); + assert_eq!( + serde_json::from_value::(shifted).unwrap(), + record + ); + record.web_port = 9090; + assert_eq!(serde_json::to_value(record).unwrap(), legacy); +} diff --git a/lib/crowdb-console-shared/tests/process_observation_ssh_test.rs b/lib/crowdb-console-shared/tests/process_observation_ssh_test.rs new file mode 100644 index 000000000..5895ad900 --- /dev/null +++ b/lib/crowdb-console-shared/tests/process_observation_ssh_test.rs @@ -0,0 +1,99 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_console_shared::{config::NodeEntry, deployment::admission::RemoteProcessObservation}; +use crowdb_protocol::mgmt::{node::NodeBinding, SystemBootstrapIdentity}; +use serde_json::json; +use std::time::Duration; + +#[path = "common/ssh_server.rs"] +mod ssh_server; + +#[tokio::test] +async fn bound_process_reader_reuses_authentication_and_discards_failed_or_cancelled_reads() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("process-observation-ssh"); + std::env::set_var("CROWDB_KV_KNOWN_HOSTS", root.path().join("known-hosts")); + let key = russh::keys::key::KeyPair::generate_ed25519().unwrap(); + let path = root.path().join("node-key"); + russh::keys::encode_pkcs8_pem(&key, std::fs::File::create(&path).unwrap()).unwrap(); + let binding = NodeBinding { + node_id: 1, + bootstrap: SystemBootstrapIdentity { + cluster_id: "bound-cluster".into(), + operation_id: "bootstrap".into(), + configuration_digest: "a".repeat(64), + }, + management_seeds: vec![], + }; + let value = json!({"node_id": 2, "cluster_id": "bound-cluster", "bootstrap": binding.bootstrap, "processes": {"kv": 42}}); + let bytes = serde_json::to_vec(&value).unwrap(); + for scenario in [ + "success", + "rejection", + "wrong-node", + "wrong-bootstrap", + "cancelled", + ] { + let reply = match scenario { + "success" => ssh_server::TestReply::Stdout(bytes.clone()), + "rejection" => ssh_server::TestReply::Failure("monitor rejected observation".into()), + "wrong-node" => { + let mut foreign = value.clone(); + foreign["node_id"] = json!(3); + ssh_server::TestReply::Stdout(serde_json::to_vec(&foreign).unwrap()) + } + "wrong-bootstrap" => { + let mut foreign = value.clone(); + foreign["bootstrap"]["operation_id"] = json!("foreign-bootstrap"); + ssh_server::TestReply::Stdout(serde_json::to_vec(&foreign).unwrap()) + } + _ => ssh_server::TestReply::DelayedStdout(bytes.clone(), Duration::from_secs(1)), + }; + let server = + ssh_server::TestSshServer::start_with_result(key.clone_public_key().unwrap(), reply).await; + let node = NodeEntry { + id: 2, + rack_id: 1, + host: "127.0.0.1".into(), + ssh_port: server.port, + ssh_user: "operator".into(), + ssh_key: Some(path.to_string_lossy().into_owned()), + ssh_password: None, + ssh_credential_ref: None, + }; + let mut reader = RemoteProcessObservation::new(node.clone(), binding.clone()).unwrap(); + if scenario == "success" { + for _ in 0..5 { + assert_eq!(reader.read().await.unwrap()["processes"]["kv"], 42); + } + assert_eq!( + server.connections(), + 1, + "one authentication for repeated fresh reads" + ); + } else if scenario == "cancelled" { + for _ in 0..2 { + assert!(tokio::time::timeout(Duration::from_millis(100), reader.read()) + .await + .is_err()); + } + assert_eq!( + server.connections(), + 2, + "a cancelled read must not retain its session" + ); + } else { + for _ in 0..2 { + assert!(reader.read().await.is_err(), "{scenario}"); + } + assert_eq!( + server.connections(), + 2, + "a rejected response must discard its session" + ); + } + let mut password = node; + password.ssh_password = Some("unused".into()); + assert!(RemoteProcessObservation::new(password, binding.clone()).is_err()); + } +} diff --git a/lib/crowdb-console-shared/tests/topology_endpoint_test.rs b/lib/crowdb-console-shared/tests/topology_endpoint_test.rs new file mode 100644 index 000000000..51592c1de --- /dev/null +++ b/lib/crowdb-console-shared/tests/topology_endpoint_test.rs @@ -0,0 +1,79 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use axum::{routing::get, Json, Router}; +use crowdb_console_shared::{ + clients::http::ServerClient, + cluster::NodeHealth, + monitor::{legacy_topology_to_node_stores, MonitorCache, NodeRecord}, +}; +use crowdb_protocol::mgmt::{GroupStatus, RemoteStatus, ReplicaStatus, StoreStatus, TopologyResponse}; + +#[tokio::test] +async fn wildcard_topology_listeners_use_the_reporting_origin_before_seeding_a_leader() { + let response = TopologyResponse { + stores: vec![ + StoreStatus { + store_id: 0, + listen_addr: Some("0.0.0.0:10100".into()), + groups: vec![GroupStatus { + group_id: 0, + leader_id: 1, + local_replica: ReplicaStatus { + id: 1, + role: "Leader".into(), + ..Default::default() + }, + remotes: vec![RemoteStatus { + id: 2, + endpoint: "192.0.2.20:10101".into(), + ..Default::default() + }], + ..Default::default() + }], + ..Default::default() + }, + StoreStatus { + store_id: 1, + listen_addr: Some("[::]:11234".into()), + ..Default::default() + }, + StoreStatus { + store_id: 2, + listen_addr: Some("192.0.2.30:12345".into()), + ..Default::default() + }, + ], + }; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!("http://localhost:{}", listener.local_addr().unwrap().port()); + let app = Router::new().route( + "/topology", + get(move || { + let response = response.clone(); + async move { Json(response) } + }), + ); + let task = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let stores = ServerClient::new(origin).unwrap().topology().await.unwrap(); + task.abort(); + assert_eq!(stores[0].listen_addr.as_deref(), Some("localhost:10100")); + assert_eq!(stores[1].listen_addr.as_deref(), Some("localhost:11234")); + assert_eq!(stores[2].listen_addr.as_deref(), Some("192.0.2.30:12345")); + assert_eq!(stores[0].groups[0].remotes[0].endpoint, "192.0.2.20:10101"); + let cache = MonitorCache::new(); + cache + .set_node_report( + 1, + NodeRecord { + health: NodeHealth::Up, + stores: legacy_topology_to_node_stores(1, &stores), + ..Default::default() + }, + ) + .await; + assert_eq!( + cache.group0_leader_endpoint().await.as_deref(), + Some("localhost:10100") + ); +} diff --git a/lib/crowdb-diskio-client/src/semantic/observation.rs b/lib/crowdb-diskio-client/src/semantic/observation.rs index 8e2fe667a..90ee1540c 100644 --- a/lib/crowdb-diskio-client/src/semantic/observation.rs +++ b/lib/crowdb-diskio-client/src/semantic/observation.rs @@ -24,6 +24,31 @@ pub struct DiskPlacement { } impl DiskTopologyView { + /// Build a metadata-only observation without opening transport connections. + #[cfg(feature = "test-util")] + #[must_use] + pub fn from_routes_for_tests(routes: Vec) -> Self { + let mut snapshot = RouteSnapshot::empty(); + snapshot.generation = 1; + for route in routes { + let endpoint: Arc = Arc::from(route.endpoint); + snapshot.routes.insert( + route.disk_id, + crate::topology::DiskRoute { + rack_id: route.rack_id, + node_id: route.node_id, + disk_group_id: route.disk_group_id, + instance_id: route.instance_id, + pool_key: Arc::from(format!("{}@{}", route.instance_id, endpoint)), + endpoint, + }, + ); + } + Self { + snapshot: Arc::new(snapshot), + } + } + /// Process-local route-cache generation, independent of catalog epochs. #[must_use] pub fn generation(&self) -> u64 { diff --git a/lib/crowdb-kv-client/src/binding/chunk_slots/publication/epochs.rs b/lib/crowdb-kv-client/src/binding/chunk_slots/publication/epochs.rs index 5b2371095..d5f0b475b 100644 --- a/lib/crowdb-kv-client/src/binding/chunk_slots/publication/epochs.rs +++ b/lib/crowdb-kv-client/src/binding/chunk_slots/publication/epochs.rs @@ -44,7 +44,9 @@ impl ChunkSlotMapClient { /// Advance this instance's owned slot epochs once before a restarted process serves. /// /// # Errors - /// Returns a publication conflict or a missing/corrupt dynamic assignment. + /// Recomputes the current assignment after `CasBusy` or `CasFailed`, up to + /// 32 attempts. Other errors, including an unreconciled `OutcomeUnknown`, + /// stop immediately. Exhausted contention returns `CasBusy` to the caller. pub async fn regrant_service_epochs(&self, instance_id: u64) -> Result<()> { for _ in 0..32 { match self.regrant_service_epochs_once(instance_id).await { @@ -74,13 +76,29 @@ impl ChunkSlotMapClient { /// No old-write draining or data-group network checks participate in publication. /// /// # Errors - /// Rejects stale publication, epoch rollback/reuse and ambiguous conflicting outcomes. + /// Returns `CasBusy` when same-head contention cannot yet be reconciled, + /// `CasFailed` for a stale source, and `OutcomeUnknown` for an unconfirmed + /// submitted outcome. Callers must reconcile the complete map before + /// deciding whether to resubmit the same intent; this method never rebases + /// a fixed publication or silently retries its mutation. Epoch rollback + /// and reuse are rejected. pub async fn publish_service_epochs(&self, next: &ChunkSlotMap) -> Result<()> { match self.publish_service_epochs_once(next).await { Err(error @ (Error::OutcomeUnknown | Error::CasBusy | Error::CasFailed { .. })) => { // A competing identical publisher can commit during preparation, // before this caller submits its CAS. Reconcile that boundary too. - let current = self.read_service_snapshot().await?; + let current = match self.read_service_snapshot().await { + Ok(current) => current, + Err(reconcile_error) if matches!(&error, Error::OutcomeUnknown) => { + tracing::warn!( + generation = next.head().generation, + %reconcile_error, + "epoch publication outcome remains unknown; reconcile the complete map before retrying" + ); + return Err(error); + } + Err(reconcile_error) => return Err(reconcile_error), + }; if current.authority().head() == next.head() && current .authority() diff --git a/lib/crowdb-kv-client/src/client/core.rs b/lib/crowdb-kv-client/src/client/core.rs index 590c23477..53a821425 100644 --- a/lib/crowdb-kv-client/src/client/core.rs +++ b/lib/crowdb-kv-client/src/client/core.rs @@ -141,16 +141,27 @@ pub struct CrowdbKvClient { impl CrowdbKvClient { #[must_use] pub fn new(config: ClientConfig) -> Self { - Self::build(config, None) + Self::build(config, None, false) + } + + /// Keep point reads independent of bulk response frames on this client's + /// existing transport, with one additional connection per endpoint. + #[must_use] + pub fn new_with_isolated_point_reads(config: ClientConfig) -> Self { + Self::build(config, None, true) } /// Create a client using an existing shared RPC transport. #[must_use] pub fn new_with_rpc_transport(config: ClientConfig, transport: Arc) -> Self { - Self::build(config, Some(transport)) + Self::build(config, Some(transport), false) } - fn build(config: ClientConfig, transport: Option>) -> Self { + fn build( + config: ClientConfig, + transport: Option>, + isolated_point_reads: bool, + ) -> Self { // A standalone client is normal at process startup. Keep the // distinction available for diagnostics without warning on each // test listener restart or each independently deployed process. @@ -174,14 +185,19 @@ impl CrowdbKvClient { metrics: Arc::new(ClientMetrics::default()), read_endpoint_policy: config.read_endpoint_policy, rpc_transport: Some(transport.unwrap_or_else(|| { - std::sync::Arc::new(crate::KvRpcTransport::with_pool_size( + let transport = crate::KvRpcTransport::with_pool_size( config.pool_size_per_endpoint, config.enable_nagle, config.quickack, config.event_write, config.send_queue_capacity, config.rpc_workers, - )) + ); + Arc::new(if isolated_point_reads { + transport.with_isolated_point_reads() + } else { + transport + }) })), } } diff --git a/lib/crowdb-kv-client/src/client/retry.rs b/lib/crowdb-kv-client/src/client/retry.rs index 55003c87f..baa80abb0 100644 --- a/lib/crowdb-kv-client/src/client/retry.rs +++ b/lib/crowdb-kv-client/src/client/retry.rs @@ -60,9 +60,12 @@ impl CrowdbKvClient { /// handled by [`Self::follow_not_leader`] before this is checked). /// Checks the structured `error_code` first, falling back to the /// string for old servers that don't set the code (default 0 = - /// `KvErrorNone`). + /// `KvErrorNone`). A removed replica may still have a cached route but no + /// local group; discover its successor instead of repeating that route. pub(crate) fn is_unknown_leader(error_code: i32, error: &str) -> bool { - error_code == KvErrorCode::KvErrorNotLeader as i32 || error == "not leader" + error_code == KvErrorCode::KvErrorNotLeader as i32 + || error == "not leader" + || error == "no kv group configured for request group_id" } /// After an [`Self::is_unknown_leader`] failure, give the election a @@ -81,7 +84,11 @@ impl CrowdbKvClient { self.metrics.record_unknown_leader_wait(); self.metrics.record_leader_query(); self.metrics.record_topology_refresh(); - if let Err(e) = self.topology.refresh().await { + if let Err(e) = self + .topology + .refresh_after_failure(store_id, group_id, endpoint) + .await + { if matches!(e, Error::NoSeeds) { warn!(error = %e, "topology refresh failed in wait_and_refresh_leader"); return endpoint.to_string(); @@ -114,7 +121,11 @@ impl CrowdbKvClient { ) -> String { self.metrics.record_topology_refresh(); let mut no_seeds = false; - if let Err(e) = self.topology.refresh().await { + if let Err(e) = self + .topology + .refresh_after_failure(store_id, group_id, current) + .await + { if matches!(e, Error::NoSeeds) { no_seeds = true; } diff --git a/lib/crowdb-kv-client/src/client/topology.rs b/lib/crowdb-kv-client/src/client/topology.rs index b8bf73d95..fee39b06b 100644 --- a/lib/crowdb-kv-client/src/client/topology.rs +++ b/lib/crowdb-kv-client/src/client/topology.rs @@ -19,6 +19,7 @@ use crate::error::{Error, Result}; use crate::ReadEndpointPolicy; mod address; +mod recovery; type GroupKey = (u64, u64); @@ -234,7 +235,7 @@ impl TopologyCache { } async fn fetch_and_merge(&self, base_generation: u64) -> Result<()> { - self.fetch_group_and_merge(base_generation, None) + self.fetch_group_and_merge(base_generation, None, None) .await .map(|_| ()) } @@ -247,7 +248,7 @@ impl TopologyCache { // A recent unrelated snapshot cannot answer a missing group's query. let generation = self.snapshot.load().generation; let result = self - .fetch_group_and_merge(generation, Some((store_id, group_id))) + .fetch_group_and_merge(generation, Some((store_id, group_id)), None) .await; *last = Instant::now(); result @@ -257,6 +258,7 @@ impl TopologyCache { &self, base_generation: u64, required_group: Option, + rejected_endpoint: Option<&str>, ) -> Result> { let seeds = self.seeds.read().unwrap().clone(); if seeds.is_empty() { @@ -287,7 +289,9 @@ impl TopologyCache { .map(|remote| remote.endpoint.clone()) } }) - .filter(|endpoint| !endpoint.is_empty()); + .filter(|endpoint| { + !endpoint.is_empty() && rejected_endpoint != Some(endpoint.as_str()) + }); if required_group.is_some() && endpoint.is_none() { continue; } diff --git a/lib/crowdb-kv-client/src/client/topology/recovery.rs b/lib/crowdb-kv-client/src/client/topology/recovery.rs new file mode 100644 index 000000000..7f7e018f3 --- /dev/null +++ b/lib/crowdb-kv-client/src/client/topology/recovery.rs @@ -0,0 +1,34 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Discovery after a cached endpoint has failed or rejected leadership. + +use super::TopologyCache; +use crate::error::Result; +use std::time::Instant; + +impl TopologyCache { + pub(in crate::client) async fn refresh_after_failure( + &self, + store_id: u64, + group_id: u64, + failed: &str, + ) -> Result<()> { + let mut last = self.refresh_gate.lock().await; + if self + .leader(store_id, group_id) + .is_some_and(|leader| leader != failed) + || last.elapsed() < self.min_refresh_interval + { + return Ok(()); + } + let generation = self.snapshot.load().generation; + // Reachability of a management seed does not establish freshness of + // its leader view. Keep looking if it repeats the rejected endpoint. + let result = self + .fetch_group_and_merge(generation, Some((store_id, group_id)), Some(failed)) + .await; + *last = Instant::now(); + result.map(|_| ()) + } +} diff --git a/lib/crowdb-kv-client/src/lib.rs b/lib/crowdb-kv-client/src/lib.rs index 6a8cfa8e7..f0aa40e1c 100644 --- a/lib/crowdb-kv-client/src/lib.rs +++ b/lib/crowdb-kv-client/src/lib.rs @@ -50,6 +50,7 @@ pub use hardware::{ }; pub use membership::{GroupMembershipClient, GroupMembershipSnapshot}; pub use metrics::{ClientMetrics, ClientMetricsSnapshot, LeaderChangeEpisode, WindowLatencySnapshot}; +pub use service::registry::DEFAULT_TTL_MS as SERVICE_INSTANCE_TTL_MS; pub use service::{ DomainMonitorClient, ServiceDiscoveryClient, ServiceRegistryClient, StreamProcessAuthority, StreamProcessContext, StreamProcessLeaseClient, StreamProcessLeaseError, StreamProcessLeaseStatus, diff --git a/lib/crowdb-kv-client/src/service/registry.rs b/lib/crowdb-kv-client/src/service/registry.rs index c119f4671..8d0fff807 100644 --- a/lib/crowdb-kv-client/src/service/registry.rs +++ b/lib/crowdb-kv-client/src/service/registry.rs @@ -29,7 +29,7 @@ const G0_STORE: u64 = 0; const G0_GROUP: u64 = 0; /// Default liveness TTL: 3× a 5s heartbeat interval = 15s. -const DEFAULT_TTL_MS: u64 = 15_000; +pub const DEFAULT_TTL_MS: u64 = 15_000; fn now_ms() -> u64 { SystemTime::now() diff --git a/lib/crowdb-kv-client/src/service/registry/diskio.rs b/lib/crowdb-kv-client/src/service/registry/diskio.rs index 7c59fd83b..bf44cd997 100644 --- a/lib/crowdb-kv-client/src/service/registry/diskio.rs +++ b/lib/crowdb-kv-client/src/service/registry/diskio.rs @@ -96,13 +96,30 @@ impl ServiceRegistryClient { { GetOutcome::Found { value, revision } => (value, revision), GetOutcome::NotFound => { + if current_scope.is_some() { + return Err(Error::Server( + "DiskIO configured scope requires a registered Rack".into(), + )); + } return self .kv .put_cas(0, 0, path.as_bytes(), payload, revision) .await - .map(|_| ()) + .map(|_| ()); } }; + // The Rack fence must precede the instance observation it protects. + // A competing first scope may have committed after our first read + // but before this Rack read; do not overwrite that accepted scope. + if self + .instance_record(path) + .await? + .as_ref() + .map_or(0, |(_, revision)| *revision) + != revision + { + continue; + } let operations = [ BatchOp::Put { key: path.as_bytes().to_vec().into(), diff --git a/lib/crowdb-kv-client/src/transport/rpc_transport.rs b/lib/crowdb-kv-client/src/transport/rpc_transport.rs index 9ddf16df8..b76d03732 100644 --- a/lib/crowdb-kv-client/src/transport/rpc_transport.rs +++ b/lib/crowdb-kv-client/src/transport/rpc_transport.rs @@ -16,7 +16,11 @@ //! window; `CrowdbKvClient` selects the transport via //! `with_rpc_transport`. +mod connections; mod owned; +mod point_reads; + +use connections::normalize_endpoint; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; @@ -59,6 +63,7 @@ pub struct KvRpcTransport { server: Arc, rpc: Arc, connections: ConnectionPoolIndex, + point_connections: Option, next_req_id: AtomicU64, } @@ -66,116 +71,16 @@ impl std::fmt::Debug for KvRpcTransport { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("KvRpcTransport") .field("next_req_id", &self.next_req_id.load(Ordering::Relaxed)) + .field("isolated_point_reads", &self.point_connections.is_some()) .finish_non_exhaustive() } } impl KvRpcTransport { - /// Create a new crowdb-rpc transport with `pool_size` connections - /// per endpoint, using the given RPC tunables and `workers` I/O - /// worker threads. The `RpcServer` is the client-side transport — - /// it does not listen but is used to establish connections to - /// remote endpoints. - #[must_use] - pub fn with_pool_size( - pool_size: usize, - enable_nagle: bool, - quickack: bool, - event_write: bool, - send_queue_capacity: u32, - workers: u32, - ) -> Self { - let server = Arc::new(RpcServer::with_engines(None, 1, workers)); - server.set_tcp_nodelay(!enable_nagle); - server.set_quickack(quickack); - server.set_event_write(event_write); - server.set_send_queue_capacity(send_queue_capacity); - server.start(); - server.register_conn_count_gauge("rpc.client.connections"); - let rpc = Arc::new(RpcClient::new()); - rpc.set_completion_pool_size(1024); - rpc.start_reaper(5_000_000_000, 500_000_000); - Self { - server, - rpc, - connections: ConnectionPoolIndex::new(pool_size, None), - next_req_id: AtomicU64::new(1), - } - } - - /// Create a new crowdb-rpc transport with a single connection per - /// endpoint and default tunables (backward-compatible). - #[must_use] - pub fn new() -> Self { - Self::with_pool_size(1, false, false, false, 4096, 2) - } - fn next_id(&self) -> u64 { self.next_req_id.fetch_add(1, Ordering::Relaxed) } - /// Clear all cached connections. Called after a server restart to - /// force reconnection on the next request. - pub fn clear_connections(&self) { - self.connections.clear(); - } - - /// Convert an `RpcError` to `Error`, dropping cached connections - /// for `endpoint` on connection-level errors (closed/reset) and - /// `Timeout`. `Timeout` can indicate a dead connection whose TCP - /// FIN has not yet been processed by the epoll worker (e.g. the - /// remote process was killed and restarted on the same port) — the - /// reaper fails the pending request but leaves the stale fd in the - /// pool, so retrying on the same connection would time out again. - /// Dropping forces `conn_for` to establish a fresh connection. - /// `SendQueueFull` is truly transient (the connection is alive but - /// the send queue is full) so the pool is preserved. Preserves the - /// endpoint in the error message. - fn map_rpc_err(&self, e: RpcError, endpoint: &str, generation: u64) -> Error { - if matches!( - e, - RpcError::ConnectionClosed | RpcError::ConnectionError | RpcError::Timeout - ) { - self.connections - .invalidate(&normalize_endpoint(endpoint), generation); - } - Error::Transport { - endpoint: endpoint.to_string(), - status: format!("rpc error: {e:?}"), - } - } - - /// Get or create a `Connection` for the given endpoint, round- - /// robining across the pool. The crowdb-rpc server listens on the - /// same port as the crowdb-rpc endpoint (no port derivation). - fn conn_for(&self, rpc_endpoint: &str) -> Result { - let normalized = normalize_endpoint(rpc_endpoint); - let (host, port) = parse_endpoint(&normalized).map_err(|e| Error::InvalidEndpoint { - endpoint: rpc_endpoint.to_string(), - reason: e, - })?; - self.connections - .get_or_try_install(&normalized, || { - let conn = self.server.connect(&host, port).map_err(|e| Error::Transport { - endpoint: rpc_endpoint.to_string(), - status: format!("rpc connect to {host}:{port}: {e:?}"), - })?; - self.rpc.attach(&conn); - Ok(conn) - }) - .map_err(|error| match error { - ConnectionPoolError::Closed => Error::Transport { - endpoint: rpc_endpoint.to_string(), - status: "connection closed during acquisition".into(), - }, - ConnectionPoolError::Connect(error) => error, - ConnectionPoolError::Capacity { max_endpoints } => Error::Transport { - endpoint: rpc_endpoint.to_string(), - status: format!("endpoint connection limit {max_endpoints} reached"), - }, - }) - } - /// Send a `Put` request via crowdb-rpc. Returns the /// `KvResponse` so the caller's retry logic is unchanged. #[allow(clippy::too_many_arguments)] @@ -296,78 +201,6 @@ impl KvRpcTransport { parse_kv_response(control.bytes()) } - /// Send a `Get` request via crowdb-rpc. - #[allow(clippy::too_many_arguments)] - pub async fn send_get( - &self, - rpc_endpoint: &str, - key: &[u8], - request_id: u64, - request_create_ms: u64, - group_id: u64, - read_mode: ReadMode, - min_slot: u64, - ) -> Result { - self.send_get_with_forwarded( - rpc_endpoint, - key, - request_id, - request_create_ms, - group_id, - read_mode, - min_slot, - false, - ) - .await - } - - /// Send a `Get` request with an explicit forwarded loop-guard flag. - #[allow(clippy::too_many_arguments)] - pub async fn send_get_with_forwarded( - &self, - rpc_endpoint: &str, - key: &[u8], - request_id: u64, - request_create_ms: u64, - group_id: u64, - read_mode: ReadMode, - min_slot: u64, - forwarded: bool, - ) -> Result { - let req_id = self.next_id(); - let conn = self.conn_for(rpc_endpoint)?; - let mut builder = FlatBufferBuilder::new(); - let fb_key = builder.create_vector(key); - let args = FBKvGetRequestArgs { - id: req_id, - rpc_create_nano: 0, - version: 1, - key: Some(fb_key), - request_id, - request_create_ms, - group_id, - read_mode: read_mode_to_fb(read_mode), - min_slot, - forwarded, - }; - let req = FBKvGetRequest::create(&mut builder, &args); - builder.finish(req, None); - let control = Buffer::from_bytes(builder.finished_data()); - let msg_type = FBMsgType::EKvGetRequest.0 as u16; - let fut = self - .rpc - .call(&self.server, &conn, req_id, control, None, msg_type) - .map_err(|e| self.map_rpc_err(e, rpc_endpoint, conn.generation()))?; - let resp = fut - .await - .map_err(|e| self.map_rpc_err(e, rpc_endpoint, conn.generation()))?; - let ctrl = resp.control.ok_or_else(|| Error::Transport { - endpoint: rpc_endpoint.to_string(), - status: "get response missing control buffer".into(), - })?; - parse_kv_response(ctrl.bytes()) - } - /// Send a `Delete` request via crowdb-rpc. #[allow(clippy::too_many_arguments)] pub async fn send_delete( @@ -1161,26 +994,3 @@ fn fb_ret_code_to_kv_error_code(code: FBKvClientRetCode) -> i32 { _ => KvErrorCode::KvErrorInternal as i32, } } - -fn normalize_endpoint(endpoint: &str) -> String { - let with_scheme = if endpoint.starts_with("http://") || endpoint.starts_with("https://") { - endpoint.to_string() - } else { - format!("http://{endpoint}") - }; - with_scheme.replacen("://0.0.0.0:", "://127.0.0.1:", 1) -} - -fn parse_endpoint(endpoint: &str) -> std::result::Result<(String, i32), String> { - let without_scheme = endpoint - .strip_prefix("http://") - .or_else(|| endpoint.strip_prefix("https://")) - .unwrap_or(endpoint); - let (host, port_str) = without_scheme - .rsplit_once(':') - .ok_or_else(|| format!("invalid endpoint: {endpoint}"))?; - let port: i32 = port_str - .parse() - .map_err(|_| format!("invalid port in endpoint: {endpoint}"))?; - Ok((host.to_string(), port)) -} diff --git a/lib/crowdb-kv-client/src/transport/rpc_transport/connections.rs b/lib/crowdb-kv-client/src/transport/rpc_transport/connections.rs new file mode 100644 index 000000000..4031b992f --- /dev/null +++ b/lib/crowdb-kv-client/src/transport/rpc_transport/connections.rs @@ -0,0 +1,152 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Shared transport owners and independently invalidated point/ordinary pools. + +use super::{ + Arc, AtomicU64, ConnectionPoolError, ConnectionPoolIndex, Error, KvRpcTransport, Result, RpcClient, + RpcError, RpcServer, SelectedConnection, +}; + +impl KvRpcTransport { + /// Create a new crowdb-rpc transport with `pool_size` connections + /// per endpoint, using the given RPC tunables and `workers` I/O + /// worker threads. The `RpcServer` is the client-side transport — + /// it does not listen but is used to establish connections to + /// remote endpoints. + #[must_use] + pub fn with_pool_size( + pool_size: usize, + enable_nagle: bool, + quickack: bool, + event_write: bool, + send_queue_capacity: u32, + workers: u32, + ) -> Self { + let server = Arc::new(RpcServer::with_engines(None, 1, workers)); + server.set_tcp_nodelay(!enable_nagle); + server.set_quickack(quickack); + server.set_event_write(event_write); + server.set_send_queue_capacity(send_queue_capacity); + server.start(); + server.register_conn_count_gauge("rpc.client.connections"); + let rpc = Arc::new(RpcClient::new()); + rpc.set_completion_pool_size(1024); + rpc.start_reaper(5_000_000_000, 500_000_000); + Self { + server, + rpc, + connections: ConnectionPoolIndex::new(pool_size, None), + point_connections: None, + next_req_id: AtomicU64::new(1), + } + } + + /// Create a new crowdb-rpc transport with a single connection per + /// endpoint and default tunables (backward-compatible). + #[must_use] + pub fn new() -> Self { + Self::with_pool_size(1, false, false, false, 4096, 2) + } + + /// Reserve one connection per endpoint for Get replies, independently of + /// scan and mutation frames. Uses the same I/O workers, correlation pool, + /// deadlines and tunables. Ordinary transports keep their existing pool. + #[must_use] + pub fn with_isolated_point_reads(mut self) -> Self { + self.point_connections = Some(ConnectionPoolIndex::new(1, None)); + self + } + + /// Clear both pools after an explicit reconnect request. + pub fn clear_connections(&self) { + self.connections.clear(); + if let Some(points) = &self.point_connections { + points.clear(); + } + } + + fn point_index(&self) -> &ConnectionPoolIndex { + self.point_connections.as_ref().unwrap_or(&self.connections) + } + + pub(super) fn map_rpc_err(&self, error: RpcError, endpoint: &str, generation: u64) -> Error { + map_error(error, endpoint, generation, &self.connections) + } + + pub(super) fn map_get_rpc_err(&self, error: RpcError, endpoint: &str, generation: u64) -> Error { + map_error(error, endpoint, generation, self.point_index()) + } + + pub(super) fn conn_for(&self, endpoint: &str) -> Result { + self.conn_for_in(endpoint, &self.connections) + } + + pub(super) fn conn_for_point(&self, endpoint: &str) -> Result { + self.conn_for_in(endpoint, self.point_index()) + } + + fn conn_for_in(&self, rpc_endpoint: &str, index: &ConnectionPoolIndex) -> Result { + let normalized = normalize_endpoint(rpc_endpoint); + let (host, port) = parse_endpoint(&normalized).map_err(|e| Error::InvalidEndpoint { + endpoint: rpc_endpoint.to_string(), + reason: e, + })?; + index + .get_or_try_install(&normalized, || { + let conn = self.server.connect(&host, port).map_err(|e| Error::Transport { + endpoint: rpc_endpoint.to_string(), + status: format!("rpc connect to {host}:{port}: {e:?}"), + })?; + self.rpc.attach(&conn); + Ok(conn) + }) + .map_err(|error| match error { + ConnectionPoolError::Closed => Error::Transport { + endpoint: rpc_endpoint.to_string(), + status: "connection closed during acquisition".into(), + }, + ConnectionPoolError::Connect(error) => error, + ConnectionPoolError::Capacity { max_endpoints } => Error::Transport { + endpoint: rpc_endpoint.to_string(), + status: format!("endpoint connection limit {max_endpoints} reached"), + }, + }) + } +} + +fn map_error(error: RpcError, endpoint: &str, generation: u64, index: &ConnectionPoolIndex) -> Error { + if matches!( + error, + RpcError::ConnectionClosed | RpcError::ConnectionError | RpcError::Timeout + ) { + index.invalidate(&normalize_endpoint(endpoint), generation); + } + Error::Transport { + endpoint: endpoint.to_string(), + status: format!("rpc error: {error:?}"), + } +} + +pub(super) fn normalize_endpoint(endpoint: &str) -> String { + let with_scheme = if endpoint.starts_with("http://") || endpoint.starts_with("https://") { + endpoint.to_string() + } else { + format!("http://{endpoint}") + }; + with_scheme.replacen("://0.0.0.0:", "://127.0.0.1:", 1) +} + +fn parse_endpoint(endpoint: &str) -> std::result::Result<(String, i32), String> { + let without_scheme = endpoint + .strip_prefix("http://") + .or_else(|| endpoint.strip_prefix("https://")) + .unwrap_or(endpoint); + let (host, port_str) = without_scheme + .rsplit_once(':') + .ok_or_else(|| format!("invalid endpoint: {endpoint}"))?; + let port: i32 = port_str + .parse() + .map_err(|_| format!("invalid port in endpoint: {endpoint}"))?; + Ok((host.to_string(), port)) +} diff --git a/lib/crowdb-kv-client/src/transport/rpc_transport/point_reads.rs b/lib/crowdb-kv-client/src/transport/rpc_transport/point_reads.rs new file mode 100644 index 000000000..6174dc7a6 --- /dev/null +++ b/lib/crowdb-kv-client/src/transport/rpc_transport/point_reads.rs @@ -0,0 +1,83 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Get requests retain their wire semantics on an optional isolated connection. + +use super::{ + parse_kv_response, read_mode_to_fb, Buffer, Error, FBKvGetRequest, FBKvGetRequestArgs, FBMsgType, + FlatBufferBuilder, KvResponse, KvRpcTransport, ReadMode, Result, +}; + +impl KvRpcTransport { + /// Send a `Get` request via crowdb-rpc. + #[allow(clippy::too_many_arguments)] + pub async fn send_get( + &self, + rpc_endpoint: &str, + key: &[u8], + request_id: u64, + request_create_ms: u64, + group_id: u64, + read_mode: ReadMode, + min_slot: u64, + ) -> Result { + self.send_get_with_forwarded( + rpc_endpoint, + key, + request_id, + request_create_ms, + group_id, + read_mode, + min_slot, + false, + ) + .await + } + + /// Send a `Get` request with an explicit forwarded loop-guard flag. + #[allow(clippy::too_many_arguments)] + pub async fn send_get_with_forwarded( + &self, + rpc_endpoint: &str, + key: &[u8], + request_id: u64, + request_create_ms: u64, + group_id: u64, + read_mode: ReadMode, + min_slot: u64, + forwarded: bool, + ) -> Result { + let req_id = self.next_id(); + let conn = self.conn_for_point(rpc_endpoint)?; + let mut builder = FlatBufferBuilder::new(); + let fb_key = builder.create_vector(key); + let args = FBKvGetRequestArgs { + id: req_id, + rpc_create_nano: 0, + version: 1, + key: Some(fb_key), + request_id, + request_create_ms, + group_id, + read_mode: read_mode_to_fb(read_mode), + min_slot, + forwarded, + }; + let req = FBKvGetRequest::create(&mut builder, &args); + builder.finish(req, None); + let control = Buffer::from_bytes(builder.finished_data()); + let msg_type = FBMsgType::EKvGetRequest.0 as u16; + let fut = self + .rpc + .call(&self.server, &conn, req_id, control, None, msg_type) + .map_err(|e| self.map_get_rpc_err(e, rpc_endpoint, conn.generation()))?; + let resp = fut + .await + .map_err(|e| self.map_get_rpc_err(e, rpc_endpoint, conn.generation()))?; + let ctrl = resp.control.ok_or_else(|| Error::Transport { + endpoint: rpc_endpoint.to_string(), + status: "get response missing control buffer".into(), + })?; + parse_kv_response(ctrl.bytes()) + } +} diff --git a/lib/crowdb-kv-client/tests/chunk_slot_publication_test.rs b/lib/crowdb-kv-client/tests/chunk_slot_publication_test.rs new file mode 100644 index 000000000..3fdd5b416 --- /dev/null +++ b/lib/crowdb-kv-client/tests/chunk_slot_publication_test.rs @@ -0,0 +1,147 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/chunk_slot_publication.rs"] +mod fixture; +#[path = "common/publication_peer.rs"] +mod peer; + +use crowdb_kv::cluster::group::PxGroup; +use crowdb_kv_client::{ChunkServiceSnapshot, ChunkSlotMapClient, ClientConfig, CrowdbKvClient, Error}; +use crowdb_protocol::{ + chunk_slot::{ChunkSlot, ChunkSlotAuthority, ChunkSlotMap}, + key::{ChunkSlotMapHeadKey, TextKey}, +}; +use fixture::TestChunkSlotPublication; +use std::{ + sync::{atomic::Ordering, Arc}, + time::Duration, +}; + +#[tokio::test] +async fn ambiguous_publication_does_not_rebase_after_busy_reconciliation() { + let peer = peer::TestPublicationPeer::start().await; + let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new(vec![]))); + kv.seed_leader(0, 0, peer.endpoint.clone()); + let maps = ChunkSlotMapClient::new(kv); + let before = maps.read_service_snapshot().await.unwrap(); + let slots: Vec<_> = ChunkSlot::all() + .filter(|slot| before.service().owner(*slot) == 1) + .map(|slot| (slot, 1)) + .collect(); + let expected = before.authority().reassign(&slots).unwrap(); + let result = maps.regrant_service_epochs(1).await; + assert!( + matches!(result, Err(Error::OutcomeUnknown)), + "ambiguous publication must not become retryable: {result:?}; submissions={}; generation={}", + peer.submissions.load(Ordering::Acquire), + peer.committed_generation.load(Ordering::Acquire) + ); + assert_eq!(peer.submissions.load(Ordering::Acquire), 1); + assert_complete(&maps.read_service_snapshot().await.unwrap(), &expected); +} + +async fn wait_for_head_admission(group: &PxGroup) { + let key = ChunkSlotMapHeadKey::Service.to_path(); + tokio::time::timeout(Duration::from_secs(1), async { + while !group.conditional_write_pending_for_tests(key.as_bytes()) { + tokio::task::yield_now().await; + } + }) + .await + .expect("publication must reach the held head admission"); +} + +fn assert_complete(actual: &ChunkServiceSnapshot, expected: &ChunkSlotMap) { + assert_eq!(actual.authority().head(), expected.head()); + assert_eq!(actual.service().head().generation, expected.head().generation); + for slot in ChunkSlot::all() { + assert_eq!(actual.authority().owner(slot), expected.owner(slot)); + assert_eq!(actual.service().owner(slot), expected.owner(slot).instance_id()); + } +} + +#[tokio::test] +async fn fixed_intent_caller_observes_busy_then_reconciles_before_explicit_republication() { + let test = TestChunkSlotPublication::start().await; + let maps = ChunkSlotMapClient::new(Arc::clone(&test.kv)); + let before = maps.read_service_snapshot().await.unwrap(); + let storage = maps.read_storage().await.unwrap(); + let revision = test.head_revision().await; + let next = before + .authority() + .reassign(&[(ChunkSlot::try_from(0).unwrap(), 3)]) + .unwrap(); + let group = test.store.get_group(0).unwrap(); + let permits = group.try_acquire_all_inflight_permits(); + let kv = Arc::clone(&test.kv); + let pending = next.clone(); + let first = + tokio::spawn(async move { ChunkSlotMapClient::new(kv).publish_service_epochs(&pending).await }); + wait_for_head_admission(&group).await; + + assert!(matches!( + maps.publish_service_epochs(&next).await, + Err(Error::CasBusy) + )); + assert!(!first.is_finished()); + assert_complete(&maps.read_service_snapshot().await.unwrap(), before.authority()); + assert_eq!(test.head_revision().await, revision); + assert_eq!(maps.read_storage().await.unwrap().bindings(), storage.bindings()); + + drop(permits); + first.await.unwrap().unwrap(); + let committed = maps.read_service_snapshot().await.unwrap(); + assert_complete(&committed, &next); + assert_eq!(maps.read_storage().await.unwrap().bindings(), storage.bindings()); + let committed_revision = test.head_revision().await; + + // The caller now has a complete committed witness for its original intent. + maps.publish_service_epochs(&next).await.unwrap(); + assert_complete(&maps.read_service_snapshot().await.unwrap(), &next); + assert_eq!(test.head_revision().await, committed_revision); +} + +#[tokio::test] +async fn restart_regrant_exhausts_busy_policy_then_recomputes_only_current_owned_slots() { + let test = TestChunkSlotPublication::start().await; + let maps = ChunkSlotMapClient::new(Arc::clone(&test.kv)); + let before = maps.read_service_snapshot().await.unwrap(); + let storage = maps.read_storage().await.unwrap(); + let revision = test.head_revision().await; + let next = before + .authority() + .reassign(&[(ChunkSlot::try_from(0).unwrap(), 3)]) + .unwrap(); + let group = test.store.get_group(0).unwrap(); + let permits = group.try_acquire_all_inflight_permits(); + let kv = Arc::clone(&test.kv); + let pending = next.clone(); + let first = + tokio::spawn(async move { ChunkSlotMapClient::new(kv).publish_service_epochs(&pending).await }); + wait_for_head_admission(&group).await; + + assert!(matches!( + maps.regrant_service_epochs(2).await, + Err(Error::CasBusy) + )); + assert!(!first.is_finished()); + assert_complete(&maps.read_service_snapshot().await.unwrap(), before.authority()); + assert_eq!(test.head_revision().await, revision); + assert_eq!(maps.read_storage().await.unwrap().bindings(), storage.bindings()); + + drop(permits); + first.await.unwrap().unwrap(); + assert_complete(&maps.read_service_snapshot().await.unwrap(), &next); + assert_eq!(maps.read_storage().await.unwrap().bindings(), storage.bindings()); + let slots: Vec<_> = ChunkSlot::all() + .filter(|slot| next.owner(*slot).instance_id() == 2) + .map(|slot| (slot, 2)) + .collect(); + let expected = next.reassign(&slots).unwrap(); + + // Restart intent is instance-scoped; it deliberately uses the current map. + maps.regrant_service_epochs(2).await.unwrap(); + assert_complete(&maps.read_service_snapshot().await.unwrap(), &expected); + assert_eq!(maps.read_storage().await.unwrap().bindings(), storage.bindings()); +} diff --git a/lib/crowdb-kv-client/tests/common/chunk_slot_publication.rs b/lib/crowdb-kv-client/tests/common/chunk_slot_publication.rs new file mode 100644 index 000000000..d16bed7d0 --- /dev/null +++ b/lib/crowdb-kv-client/tests/common/chunk_slot_publication.rs @@ -0,0 +1,66 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use crowdb_kv::cluster::{ + group::PxGroup, kv_server::KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, +}; +use crowdb_kv_client::{ChunkSlotMapClient, ClientConfig, CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_protocol::{ + chunk_slot::{ChunkSlotBootstrap, ChunkStorageGroup}, + key::{ChunkSlotMapHeadKey, TextKey}, +}; + +pub struct TestChunkSlotPublication { + pub store: Arc, + pub kv: Arc, +} + +impl TestChunkSlotPublication { + pub async fn start() -> Self { + let store = Arc::new(PxKvStore::new(0, "127.0.0.1:0".parse().unwrap())); + for group in [0, 1] { + store.add_group(PxGroup::new( + group, + PxLocalReplica::new(1, PxLocalReplicaRole::Leader), + )); + } + store.start().await.unwrap(); + let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new(vec![]))); + for group in [0, 1] { + kv.seed_leader(0, group, store.listen_addr().unwrap().to_string()); + } + let maps = ChunkSlotMapClient::new(Arc::clone(&kv)); + maps.initialize_layout(&ChunkSlotBootstrap { + service_instances: vec![1, 2], + storage_groups: vec![ChunkStorageGroup { + store_id: 0, + group_id: 1, + }], + }) + .await + .unwrap(); + maps.initialize_service_epochs().await.unwrap(); + Self { store, kv } + } + + pub async fn head_revision(&self) -> u64 { + let key = ChunkSlotMapHeadKey::Service.to_path(); + match self + .kv + .get(0, 0, key.as_bytes(), ReadMode::Linearizable, None) + .await + .unwrap() + { + GetOutcome::Found { revision, .. } => revision, + GetOutcome::NotFound => panic!("service head disappeared"), + } + } +} + +impl Drop for TestChunkSlotPublication { + fn drop(&mut self) { + self.store.stop(); + } +} diff --git a/lib/crowdb-kv-client/tests/common/point_read_peer.rs b/lib/crowdb-kv-client/tests/common/point_read_peer.rs new file mode 100644 index 000000000..3939bbe3c --- /dev/null +++ b/lib/crowdb-kv-client/tests/common/point_read_peer.rs @@ -0,0 +1,219 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_protocol::{ + fb::FBMsgType, + kv_client_fb::{ + FBKvClientRetCode, FBKvGetRequest, FBKvResponse, FBKvResponseArgs, FBKvScanItem, FBKvScanItemArgs, + FBKvScanRequest, FBKvScanResponse, FBKvScanResponseArgs, + }, +}; +use flatbuffers::FlatBufferBuilder; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt, ReadHalf, WriteHalf}, + net::{TcpListener, TcpStream}, + sync::{mpsc, Notify}, + task::{JoinHandle, JoinSet}, +}; + +#[derive(Clone, Copy)] +pub enum TestPointReply { + Ready, + Rejected, + Held, + ClosedOnce, +} + +#[derive(Default)] +pub struct TestPointState { + pub bulk_started: Notify, + pub release_bulk: Notify, + pub accepted: AtomicUsize, + pub scans: AtomicUsize, + pub gets: AtomicUsize, +} + +pub struct TestPointReadPeer { + pub endpoint: String, + pub state: Arc, + worker: JoinHandle<()>, +} + +struct Reply { + bytes: Vec, + hold: bool, +} + +impl TestPointReadPeer { + pub async fn start(reply: TestPointReply) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = listener.local_addr().unwrap().to_string(); + let state = Arc::new(TestPointState::default()); + let worker = tokio::spawn({ + let state = state.clone(); + async move { + let mut connections = JoinSet::new(); + loop { + tokio::select! { + value=listener.accept()=>{ + let(stream,_)=value.unwrap(); + state.accepted.fetch_add(1,Ordering::Release); + connections.spawn(serve(stream,state.clone(),reply)); + }, + result=connections.join_next(),if !connections.is_empty()=>{result.unwrap().unwrap();} + } + } + } + }); + Self { + endpoint, + state, + worker, + } + } +} + +impl Drop for TestPointReadPeer { + fn drop(&mut self) { + self.worker.abort(); + } +} + +async fn serve(stream: TcpStream, state: Arc, reply: TestPointReply) { + let (reader, writer) = tokio::io::split(stream); + let (tx, rx) = mpsc::channel(8); + tokio::join!( + read_requests(reader, tx, state.clone(), reply), + write_replies(writer, rx, state) + ); +} + +async fn read_requests( + mut reader: ReadHalf, + tx: mpsc::Sender, + state: Arc, + reply: TestPointReply, +) { + loop { + let mut header = [0; 14]; + if reader.read_exact(&mut header).await.is_err() { + return; + } + assert_eq!(u16::from_le_bytes(header[..2].try_into().unwrap()), 0xca70); + let kind = u16::from_le_bytes(header[2..4].try_into().unwrap()); + let length = u32::from_le_bytes(header[4..8].try_into().unwrap()) as usize; + assert!(length <= 4096); + assert_eq!(&header[8..12], &[0; 4]); + let mut control = vec![0; length]; + reader.read_exact(&mut control).await.unwrap(); + let response = if kind == 0xfffe { + assert_eq!(control.len(), 160); + let mut capabilities = vec![0; 160]; + capabilities[..4].copy_from_slice(&[0x44, 0x52, 4, 0]); + capabilities[4..12].copy_from_slice(&1_u64.to_le_bytes()); + frame(kind, &capabilities, false) + } else if kind == FBMsgType::EKvScanRequest.0 as u16 { + let id = flatbuffers::root::(&control).unwrap().id(); + let hold = state.scans.fetch_add(1, Ordering::AcqRel) == 0; + scan_reply(id, hold) + } else { + assert_eq!(kind, FBMsgType::EKvGetRequest.0 as u16); + let id = flatbuffers::root::(&control).unwrap().id(); + let index = state.gets.fetch_add(1, Ordering::AcqRel); + if matches!(reply, TestPointReply::ClosedOnce) && index == 0 { + return; + } + if matches!(reply, TestPointReply::Held) { + continue; + } + point_reply(id, reply) + }; + if tx.send(response).await.is_err() { + return; + } + } +} + +async fn write_replies( + mut writer: WriteHalf, + mut rx: mpsc::Receiver, + state: Arc, +) { + while let Some(reply) = rx.recv().await { + if reply.hold { + if writer.write_all(&reply.bytes[..15]).await.is_err() { + return; + } + state.bulk_started.notify_one(); + state.release_bulk.notified().await; + if writer.write_all(&reply.bytes[15..]).await.is_err() { + return; + } + } else if writer.write_all(&reply.bytes).await.is_err() { + return; + } + } +} + +fn frame(kind: u16, control: &[u8], hold: bool) -> Reply { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&0xca70_u16.to_le_bytes()); + bytes.extend_from_slice(&kind.to_le_bytes()); + bytes.extend_from_slice(&u32::try_from(control.len()).unwrap().to_le_bytes()); + bytes.extend_from_slice(&0_u32.to_le_bytes()); + bytes.extend_from_slice(&[14, 0]); + bytes.extend_from_slice(control); + Reply { bytes, hold } +} + +fn scan_reply(id: u64, hold: bool) -> Reply { + let mut builder = FlatBufferBuilder::new(); + let key = builder.create_vector(b"/bulk/value"); + let value = builder.create_vector(&vec![7_u8; 65536]); + let item = FBKvScanItem::create( + &mut builder, + &FBKvScanItemArgs { + key: Some(key), + value: Some(value), + ..Default::default() + }, + ); + let items = builder.create_vector(&[item]); + let response = FBKvScanResponse::create( + &mut builder, + &FBKvScanResponseArgs { + id, + version: 1, + ok: true, + items: Some(items), + ..Default::default() + }, + ); + builder.finish(response, None); + frame(FBMsgType::EKvScanResponse.0 as u16, builder.finished_data(), hold) +} + +fn point_reply(id: u64, reply: TestPointReply) -> Reply { + let mut builder = FlatBufferBuilder::new(); + let response = FBKvResponse::create( + &mut builder, + &FBKvResponseArgs { + id, + version: 1, + ok: matches!(reply, TestPointReply::Ready | TestPointReply::ClosedOnce), + not_found: matches!(reply, TestPointReply::Ready | TestPointReply::ClosedOnce), + ret_code: if matches!(reply, TestPointReply::Rejected) { + FBKvClientRetCode::Unavailable + } else { + FBKvClientRetCode::Success + }, + ..Default::default() + }, + ); + builder.finish(response, None); + frame(FBMsgType::EKvResponse.0 as u16, builder.finished_data(), false) +} diff --git a/lib/crowdb-kv-client/tests/common/publication_peer.rs b/lib/crowdb-kv-client/tests/common/publication_peer.rs new file mode 100644 index 000000000..a099c8c96 --- /dev/null +++ b/lib/crowdb-kv-client/tests/common/publication_peer.rs @@ -0,0 +1,360 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_protocol::{ + chunk_slot::{ + ChunkServiceIncarnation, ChunkSlotAuthority, ChunkSlotBinding, ChunkSlotBootstrap, ChunkStorageGroup, + }, + fb::FBMsgType, + key::{ + ChunkServiceAuthorityKey, ChunkServiceSlotsKey, ChunkSlotMapHeadKey, ChunkStorageSlotsKey, TextKey, + }, + kv_client_fb::{ + FBKvBatchWriteRequest, FBKvClientRetCode, FBKvGetRequest, FBKvResponse, FBKvResponseArgs, + FBKvScanItem, FBKvScanItemArgs, FBKvScanRequest, FBKvScanResponse, FBKvScanResponseArgs, + }, +}; +use flatbuffers::FlatBufferBuilder; +use std::{ + collections::BTreeMap, + sync::{ + atomic::{AtomicU64, AtomicUsize, Ordering}, + Arc, + }, +}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::{TcpListener, TcpStream}, + sync::{mpsc, oneshot}, + task::{JoinHandle, JoinSet}, +}; + +pub struct TestPublicationPeer { + pub endpoint: String, + pub submissions: Arc, + pub committed_generation: Arc, + worker: JoinHandle<()>, +} + +struct Request { + kind: u16, + control: Vec, + response: oneshot::Sender>, +} + +struct Records { + values: BTreeMap, (u64, Vec)>, + revision: u64, + moving_head_reads: usize, + submissions: Arc, + committed_generation: Arc, +} + +impl TestPublicationPeer { + pub async fn start() -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = listener.local_addr().unwrap().to_string(); + let submissions = Arc::new(AtomicUsize::new(0)); + let committed_generation = Arc::new(AtomicU64::new(2)); + let mut records = Records::new(submissions.clone(), committed_generation.clone()); + let (tx, mut rx) = mpsc::channel::(16); + let worker = tokio::spawn(async move { + let mut connections = JoinSet::new(); + loop { + tokio::select! { + accepted=listener.accept()=>{ + let (stream,_)=accepted.unwrap(); + connections.spawn(serve(stream,tx.clone())); + } + Some(request)=rx.recv()=>{ + let response=records.reply(request.kind,&request.control); + let _=request.response.send(response); + } + completed=connections.join_next(),if !connections.is_empty()=>{ + completed.unwrap().unwrap(); + } + } + } + }); + Self { + endpoint, + submissions, + committed_generation, + worker, + } + } +} + +impl Drop for TestPublicationPeer { + fn drop(&mut self) { + self.worker.abort(); + } +} + +impl Records { + fn new(submissions: Arc, committed_generation: Arc) -> Self { + let bootstrap = ChunkSlotBootstrap { + service_instances: vec![1, 2], + storage_groups: vec![ChunkStorageGroup { + store_id: 0, + group_id: 1, + }], + }; + let service = bootstrap.service_map().unwrap(); + let storage = bootstrap.storage_map().unwrap(); + let mut values = BTreeMap::new(); + let mut head = service.head().clone(); + head.generation = 2; + for binding in service.bindings() { + let mut binding = binding.clone(); + binding.generation = 2; + values.insert( + ChunkServiceSlotsKey { + instance_id: binding.owner, + } + .to_path() + .into_bytes(), + (100, serde_json::to_vec(&binding).unwrap()), + ); + let authority = ChunkSlotAuthority::new( + binding.owner, + ChunkServiceIncarnation::try_from([1; 16]).unwrap(), + 1, + ) + .unwrap(); + let binding = ChunkSlotBinding { + generation: 2, + owner: authority, + slots: binding.slots, + }; + values.insert( + ChunkServiceAuthorityKey { authority }.to_path().into_bytes(), + (100, serde_json::to_vec(&binding).unwrap()), + ); + } + for key in [ChunkSlotMapHeadKey::Service, ChunkSlotMapHeadKey::Authority] { + values.insert( + key.to_path().into_bytes(), + (100, serde_json::to_vec(&head).unwrap()), + ); + } + for binding in storage.bindings() { + values.insert( + ChunkStorageSlotsKey { + store_id: binding.owner.store_id, + group_id: binding.owner.group_id, + } + .to_path() + .into_bytes(), + (100, serde_json::to_vec(binding).unwrap()), + ); + } + values.insert( + ChunkSlotMapHeadKey::Storage.to_path().into_bytes(), + (100, serde_json::to_vec(storage.head()).unwrap()), + ); + Self { + values, + revision: 100, + moving_head_reads: 0, + submissions, + committed_generation, + } + } + + fn reply(&mut self, kind: u16, control: &[u8]) -> Vec { + if kind == FBMsgType::EKvBatchWriteRequest.0 as u16 { + return self.batch(control); + } + if kind == FBMsgType::EKvGetRequest.0 as u16 { + let request = flatbuffers::root::(control).unwrap(); + assert_eq!(request.group_id(), 0); + let key = request.key().unwrap().bytes(); + if key == ChunkSlotMapHeadKey::Service.to_path().as_bytes() && self.moving_head_reads > 0 { + // A competing cohort reservation can rewrite an unchanged service head. + self.moving_head_reads -= 1; + self.revision += 1; + self.values.get_mut(key).unwrap().0 = self.revision; + } + let found = self.values.get(key); + return point( + request.id(), + true, + self.revision, + found, + FBKvClientRetCode::Success, + ); + } + assert_eq!(kind, FBMsgType::EKvScanRequest.0 as u16); + let request = flatbuffers::root::(control).unwrap(); + assert_eq!(request.group_id(), 0); + let cutoff = if request.scan_cutoff() == 0 { + self.revision + } else { + request.scan_cutoff() + }; + let prefix = request.prefix().unwrap().bytes(); + let after = request.start_after().map_or(&[][..], |key| key.bytes()); + let rows: Vec<_> = self + .values + .iter() + .filter(|(key, (revision, _))| { + key.starts_with(prefix) && key.as_slice() > after && *revision <= cutoff + }) + .collect(); + let mut builder = FlatBufferBuilder::new(); + let items: Vec<_> = rows + .into_iter() + .map(|(key, (revision, value))| { + let key = builder.create_vector(key); + let value = builder.create_vector(value); + FBKvScanItem::create( + &mut builder, + &FBKvScanItemArgs { + key: Some(key), + value: Some(value), + commit_slot: *revision, + }, + ) + }) + .collect(); + let items = builder.create_vector(&items); + let response = FBKvScanResponse::create( + &mut builder, + &FBKvScanResponseArgs { + id: request.id(), + version: 1, + ok: true, + items: Some(items), + read_slot: self.revision, + scan_cutoff: cutoff, + ..Default::default() + }, + ); + builder.finish(response, None); + frame(FBMsgType::EKvScanResponse.0 as u16, builder.finished_data()) + } + + fn batch(&mut self, control: &[u8]) -> Vec { + let request = flatbuffers::root::(control).unwrap(); + assert_eq!(request.group_id(), 0); + let precondition = request.precondition().unwrap(); + let key = precondition.key().unwrap().bytes(); + assert_eq!(key, ChunkSlotMapHeadKey::Service.to_path().as_bytes()); + assert_eq!(precondition.expected_revision(), self.values[key].0); + self.revision += 1; + for item in request.items().unwrap() { + let key = item.key().unwrap().bytes().to_vec(); + if item.is_delete() { + self.values.remove(&key); + } else { + self.values + .insert(key, (self.revision, item.value().unwrap().bytes().to_vec())); + } + } + let first = self.submissions.fetch_add(1, Ordering::AcqRel) == 0; + let head = &self.values[ChunkSlotMapHeadKey::Service.to_path().as_bytes()].1; + self.committed_generation.store( + serde_json::from_slice::(head).unwrap()["generation"] + .as_u64() + .unwrap(), + Ordering::Release, + ); + if first { + // The mutation committed, but its caller received no confirmed outcome. + // Nine head changes exhaust read_service's four consistent-read attempts. + self.moving_head_reads = 9; + } + point( + request.id(), + !first, + self.revision, + None, + if first { + FBKvClientRetCode::OutcomeUnknown + } else { + FBKvClientRetCode::Success + }, + ) + } +} + +fn point( + id: u64, + ok: bool, + read_slot: u64, + found: Option<&(u64, Vec)>, + code: FBKvClientRetCode, +) -> Vec { + let mut builder = FlatBufferBuilder::new(); + let value = found.map(|(_, value)| builder.create_vector(value)); + let response = FBKvResponse::create( + &mut builder, + &FBKvResponseArgs { + id, + version: 1, + ok, + revision: found.map_or(read_slot, |(revision, _)| *revision), + value, + not_found: ok && found.is_none(), + ret_code: code, + read_slot, + ..Default::default() + }, + ); + builder.finish(response, None); + frame(FBMsgType::EKvResponse.0 as u16, builder.finished_data()) +} + +fn frame(kind: u16, control: &[u8]) -> Vec { + let mut bytes = Vec::with_capacity(control.len() + 14); + bytes.extend_from_slice(&0xca70_u16.to_le_bytes()); + bytes.extend_from_slice(&kind.to_le_bytes()); + bytes.extend_from_slice(&u32::try_from(control.len()).unwrap().to_le_bytes()); + bytes.extend_from_slice(&0_u32.to_le_bytes()); + bytes.extend_from_slice(&[14, 0]); + bytes.extend_from_slice(control); + bytes +} + +async fn serve(mut stream: TcpStream, tx: mpsc::Sender) { + loop { + let mut header = [0; 14]; + if stream.read_exact(&mut header).await.is_err() { + return; + } + assert_eq!(u16::from_le_bytes(header[..2].try_into().unwrap()), 0xca70); + assert_eq!(&header[8..12], &[0; 4]); + let kind = u16::from_le_bytes(header[2..4].try_into().unwrap()); + let length = u32::from_le_bytes(header[4..8].try_into().unwrap()) as usize; + assert!(length <= 65536); + let mut control = vec![0; length]; + stream.read_exact(&mut control).await.unwrap(); + let reply = if kind == 0xfffe { + let mut capabilities = vec![0; 160]; + capabilities[..4].copy_from_slice(&[0x44, 0x52, 4, 0]); + capabilities[4..12].copy_from_slice(&1_u64.to_le_bytes()); + frame(kind, &capabilities) + } else { + let (response, rx) = oneshot::channel(); + if tx + .send(Request { + kind, + control, + response, + }) + .await + .is_err() + { + return; + } + let Ok(response) = rx.await else { + return; + }; + response + }; + if stream.write_all(&reply).await.is_err() { + return; + } + } +} diff --git a/lib/crowdb-kv-client/tests/common/topology_recovery.rs b/lib/crowdb-kv-client/tests/common/topology_recovery.rs new file mode 100644 index 000000000..c15614182 --- /dev/null +++ b/lib/crowdb-kv-client/tests/common/topology_recovery.rs @@ -0,0 +1,87 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use axum::{routing::get, Json, Router}; +use crowdb_kv::cluster::{ + group::PxGroup, kv_server::KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, +}; +use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use std::{ + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; + +pub struct TestDiscovery { + stores: Vec>, + servers: Vec>, + pub client: Arc, + pub fetches: Arc, +} + +impl TestDiscovery { + pub async fn start(transport_failure: bool, replacement: bool) -> Self { + let mut stores = Vec::new(); + for (id, role) in [(1, PxLocalReplicaRole::Leader), (2, PxLocalReplicaRole::Follower)] { + let store = Arc::new(PxKvStore::new(11, "127.0.0.1:0".parse().unwrap())); + store.add_group(PxGroup::new(1, PxLocalReplica::new(id, role))); + store.start().await.unwrap(); + stores.push(store); + } + let mut stale = stores[0].status(); + let failed = if transport_failure { + "127.0.0.1:1".to_owned() + } else { + stores[1].listen_addr().unwrap().to_string() + }; + stale.listen_addr = Some(failed.clone()); + let mut seeds = Vec::new(); + let mut servers = Vec::new(); + let fetches = Arc::new(AtomicUsize::new(0)); + for body in [ + stale.clone(), + if replacement { stores[0].status() } else { stale }, + ] { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + seeds.push(format!("http://{}", listener.local_addr().unwrap())); + let requests = Arc::clone(&fetches); + servers.push(tokio::spawn(async move { + let app = Router::new().route( + "/topology", + get(move || { + let body = body.clone(); + requests.fetch_add(1, Ordering::Relaxed); + async move { Json(serde_json::json!({"stores": [body]})) } + }), + ); + axum::serve(listener, app).await.unwrap(); + })); + } + let mut config = ClientConfig::new(seeds); + config.retry.max_retries = 2; + config.retry.unknown_leader_wait = Duration::from_millis(10); + config.retry.backoff_base = Duration::from_secs(1); + let client = Arc::new(CrowdbKvClient::new(config)); + client.seed_leader(11, 1, failed); + Self { + stores, + servers, + client, + fetches, + } + } + + pub async fn stop(&self) { + for server in &self.servers { + server.abort(); + } + for store in &self.stores { + store.stop(); + } + for store in &self.stores { + store.join().await; + } + } +} diff --git a/lib/crowdb-kv-client/tests/conditional_retry_test.rs b/lib/crowdb-kv-client/tests/conditional_retry_test.rs index 6ac2a0155..cdc46190e 100644 --- a/lib/crowdb-kv-client/tests/conditional_retry_test.rs +++ b/lib/crowdb-kv-client/tests/conditional_retry_test.rs @@ -8,6 +8,48 @@ use conditional_servers::TestServers; use crowdb_kv::cluster::kv_server::KvServer; use crowdb_kv_client::{BatchOp, CrowdbKvClient, Error, KvRpcTransport}; +#[tokio::test] +async fn a_removed_replica_refreshes_its_stale_route_before_reading_or_writing() { + let servers = TestServers::start(true).await; + assert!(servers.follower.remove_group(1)); + servers + .client() + .put(1, 1, b"value", b"retained", None) + .await + .unwrap(); + let value = servers + .client() + .get(1, 1, b"value", crowdb_kv_client::ReadMode::Linearizable, None) + .await + .unwrap(); + assert!( + matches!(value, crowdb_kv_client::GetOutcome::Found { value, .. } if value.as_ref() == b"retained") + ); + servers + .client() + .put_cas(1, 1, b"conditional", b"value", 0) + .await + .unwrap(); + let scan = servers + .client() + .scan( + 1, + 1, + b"value", + b"", + b"", + 10, + crowdb_kv_client::ReadMode::Linearizable, + None, + false, + None, + ) + .await + .unwrap(); + assert_eq!(scan.items.len(), 1); + servers.client().delete(1, 1, b"value", None).await.unwrap(); +} + #[tokio::test] async fn conditional_writes_discover_leader_after_explicit_rejection_without_a_hint() { let servers = TestServers::start(true).await; diff --git a/lib/crowdb-kv-client/tests/point_read_isolation_test.rs b/lib/crowdb-kv-client/tests/point_read_isolation_test.rs new file mode 100644 index 000000000..4dcf9f5c8 --- /dev/null +++ b/lib/crowdb-kv-client/tests/point_read_isolation_test.rs @@ -0,0 +1,170 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_kv::rpc::{KvErrorCode, KvResponse}; +use crowdb_kv_client::{ClientConfig, CrowdbKvClient, Error, GetOutcome, KvRpcTransport, ReadMode, Result}; +use std::{ + sync::{atomic::Ordering, Arc}, + time::Duration, +}; +use tokio::task::JoinHandle; + +#[path = "common/point_read_peer.rs"] +mod point_read_peer; +use point_read_peer::{TestPointReadPeer, TestPointReply}; + +const BUDGET: Duration = Duration::from_millis(750); + +async fn scan_value(transport: &KvRpcTransport, endpoint: &str) { + let page = transport + .send_scan( + endpoint, + b"/bulk/", + b"", + b"", + 1, + 1, + 0, + 1, + ReadMode::Linearizable, + 0, + false, + false, + 0, + false, + 0, + ) + .await + .unwrap(); + assert_eq!(page.items.len(), 1); + assert_eq!(page.items[0].value.len(), 65536); +} + +fn start_scan(transport: Arc, endpoint: String) -> JoinHandle<()> { + tokio::spawn(async move { scan_value(&transport, &endpoint).await }) +} + +async fn finish_scan(peer: &TestPointReadPeer, scan: JoinHandle<()>) { + peer.state.release_bulk.notify_one(); + tokio::time::timeout(BUDGET, scan).await.unwrap().unwrap(); +} + +async fn point(transport: &KvRpcTransport, endpoint: &str) -> Result { + transport + .send_get(endpoint, b"/chunk/point", 2, 0, 1, ReadMode::Linearizable, 0) + .await +} + +#[tokio::test] +async fn point_read_finishes_while_a_scan_frame_is_incomplete() { + let peer = TestPointReadPeer::start(TestPointReply::Ready).await; + let client = Arc::new(CrowdbKvClient::new_with_isolated_point_reads(ClientConfig::new( + vec![], + ))); + client.seed_leader(0, 1, peer.endpoint.clone()); + let scan = tokio::spawn({ + let client = client.clone(); + async move { + let page = client + .scan( + 0, + 1, + b"/bulk/", + b"", + b"", + 1, + ReadMode::Linearizable, + None, + false, + None, + ) + .await + .unwrap(); + assert_eq!(page.items[0].1.len(), 65536); + } + }); + tokio::time::timeout(BUDGET, peer.state.bulk_started.notified()) + .await + .unwrap(); + let get = tokio::time::timeout( + BUDGET, + client.get(0, 1, b"/chunk/point", ReadMode::Linearizable, None), + ) + .await; + let received = peer.state.gets.load(Ordering::Acquire); + finish_scan(&peer, scan).await; + assert_eq!( + received, 1, + "the peer received the point read while its scan response was held" + ); + assert!(matches!(get, Ok(Ok(GetOutcome::NotFound))), + "a real point read must finish within its original deadline despite an incomplete scan frame: {get:?}"); + assert_eq!(peer.state.accepted.load(Ordering::Acquire), 2); +} + +#[tokio::test] +async fn ordinary_transport_keeps_the_shared_pool_without_opt_in() { + let peer = TestPointReadPeer::start(TestPointReply::Ready).await; + let transport = Arc::new(KvRpcTransport::new()); + let scan = start_scan(transport.clone(), peer.endpoint.clone()); + tokio::time::timeout(BUDGET, peer.state.bulk_started.notified()) + .await + .unwrap(); + let get = tokio::time::timeout(BUDGET, point(&transport, &peer.endpoint)).await; + finish_scan(&peer, scan).await; + assert!( + get.is_err(), + "the control must retain shared-frame response ordering: {get:?}" + ); + assert_eq!(peer.state.gets.load(Ordering::Acquire), 1); + assert_eq!(peer.state.accepted.load(Ordering::Acquire), 1); +} + +#[tokio::test] +async fn isolated_point_reads_preserve_rejection() { + let peer = TestPointReadPeer::start(TestPointReply::Rejected).await; + let transport = KvRpcTransport::new().with_isolated_point_reads(); + let result = tokio::time::timeout(BUDGET, point(&transport, &peer.endpoint)) + .await + .unwrap() + .unwrap(); + assert!(!result.ok); + assert!(!result.not_found); + assert_eq!(result.error_code, KvErrorCode::KvErrorUnavailable as i32); + assert_eq!(peer.state.gets.load(Ordering::Acquire), 1); +} + +#[tokio::test] +async fn isolated_point_reads_keep_the_original_deadline() { + let peer = TestPointReadPeer::start(TestPointReply::Held).await; + let transport = KvRpcTransport::new().with_isolated_point_reads(); + assert!(tokio::time::timeout(BUDGET, point(&transport, &peer.endpoint)) + .await + .is_err()); + assert_eq!(peer.state.gets.load(Ordering::Acquire), 1); +} + +#[tokio::test] +async fn failed_point_generation_does_not_evict_a_healthy_scan_pool() { + let peer = TestPointReadPeer::start(TestPointReply::ClosedOnce).await; + let transport = Arc::new(KvRpcTransport::new().with_isolated_point_reads()); + let scan = start_scan(transport.clone(), peer.endpoint.clone()); + tokio::time::timeout(BUDGET, peer.state.bulk_started.notified()) + .await + .unwrap(); + let lost = tokio::time::timeout(BUDGET, point(&transport, &peer.endpoint)).await; + let recovered = tokio::time::timeout(BUDGET, point(&transport, &peer.endpoint)).await; + finish_scan(&peer, scan).await; + assert!(matches!(lost, Ok(Err(Error::Transport { .. }))), "{lost:?}"); + assert!(recovered.unwrap().unwrap().not_found); + scan_value(&transport, &peer.endpoint).await; + assert_eq!(peer.state.accepted.load(Ordering::Acquire), 3); + transport.clear_connections(); + assert!(point(&transport, &peer.endpoint).await.unwrap().not_found); + scan_value(&transport, &peer.endpoint).await; + assert_eq!( + peer.state.accepted.load(Ordering::Acquire), + 5, + "explicit clear must replace both pools; failed point reads must replace only their own generation" + ); +} diff --git a/lib/crowdb-kv-client/tests/storage_exclusion_test.rs b/lib/crowdb-kv-client/tests/storage_exclusion_test.rs index 86c667c9a..08568c263 100644 --- a/lib/crowdb-kv-client/tests/storage_exclusion_test.rs +++ b/lib/crowdb-kv-client/tests/storage_exclusion_test.rs @@ -202,6 +202,48 @@ async fn immutable_configured_scope_covers_disks_before_usage_advertisements_cat ); } +#[tokio::test] +async fn configured_scope_waits_for_its_rack_membership_fence() { + use crowdb_protocol::common::{DiskioHardwareScope, NodeValue, RackValue}; + let fixture = TestStorage::start().await; + let hardware = &fixture.hardware; + let svc = ServiceRegistryClient::from_shared(hardware.shared_kv()); + let scope = DiskioHardwareScope { + version: 1, + rack_id: 99, + node_id: 99, + disk_group_id: 0, + }; + assert!( + svc.heartbeat_diskio_scoped(99, "pending:7200", &scope, &[], &[]) + .await + .is_err(), + "a first scope cannot publish before its Rack admission fence exists" + ); + let pinned = svc.read_instance("diskio", 99).await.unwrap().unwrap(); + assert!(pinned.extra.unwrap().diskdb.unwrap().diskio_scope.is_none()); + hardware + .add_rack( + 99, + &RackValue { + node_ids: vec![99], + ..Default::default() + }, + ) + .await + .unwrap(); + hardware.add_node(99, 99, &NodeValue::default()).await.unwrap(); + svc.heartbeat_diskio_scoped(99, "ready:7200", &scope, &[], &[]) + .await + .unwrap(); + let mut other = scope.clone(); + other.disk_group_id = 3; + assert!(svc + .heartbeat_diskio_scoped(99, "other:7200", &other, &[], &[]) + .await + .is_err()); +} + #[tokio::test] async fn simultaneous_first_scope_selection_pins_one_configuration() { use crowdb_protocol::common::DiskioHardwareScope; @@ -215,9 +257,24 @@ async fn simultaneous_first_scope_selection_pins_one_configuration() { }; let mut second = first.clone(); second.disk_group_id = 3; - let (a, b) = tokio::join!( - svc.heartbeat_diskio_scoped(7, "a:7200", &first, &[], &[]), - svc.heartbeat_diskio_scoped(7, "b:7200", &second, &[], &[]) - ); - assert_ne!(a.is_ok(), b.is_ok(), "a={a:?}, b={b:?}"); + for instance_id in 7..39 { + let (a, b) = tokio::join!( + svc.heartbeat_diskio_scoped(instance_id, "a:7200", &first, &[], &[]), + svc.heartbeat_diskio_scoped(instance_id, "b:7200", &second, &[], &[]) + ); + assert_ne!(a.is_ok(), b.is_ok(), "instance={instance_id}, a={a:?}, b={b:?}"); + let (winner, loser) = if a.is_ok() { + (&first, &second) + } else { + (&second, &first) + }; + assert!(svc + .heartbeat_diskio_scoped(instance_id, "winner:7200", winner, &[], &[]) + .await + .is_ok()); + assert!(svc + .heartbeat_diskio_scoped(instance_id, "loser:7200", loser, &[], &[]) + .await + .is_err()); + } } diff --git a/lib/crowdb-kv-client/tests/topology_retry_discovery_test.rs b/lib/crowdb-kv-client/tests/topology_retry_discovery_test.rs new file mode 100644 index 000000000..332da0162 --- /dev/null +++ b/lib/crowdb-kv-client/tests/topology_retry_discovery_test.rs @@ -0,0 +1,88 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/topology_recovery.rs"] +mod topology_recovery; + +use crowdb_kv_client::{Error, GetOutcome, ReadMode}; +use std::time::Duration; +use topology_recovery::TestDiscovery; + +#[tokio::test] +async fn a_reachable_stale_seed_cannot_hide_a_replacement_after_transport_failure() { + let test = TestDiscovery::start(true, true).await; + let result = tokio::time::timeout( + Duration::from_millis(500), + test.client.put(11, 1, b"recovered", b"retained", None), + ) + .await; + let read = if matches!(result, Ok(Ok(_))) { + Some( + test.client + .get(11, 1, b"recovered", ReadMode::Linearizable, None) + .await, + ) + } else { + None + }; + test.stop().await; + assert!( + matches!(result, Ok(Ok(_))), + "stale first seed retained failed route/backoff: {result:?}" + ); + assert!(matches!(read, Some(Ok(GetOutcome::Found { value, .. })) if value.as_ref() == b"retained")); +} + +#[tokio::test] +async fn a_reachable_stale_seed_cannot_hide_a_replacement_after_no_hint_rejection() { + let test = TestDiscovery::start(false, true).await; + let result = tokio::time::timeout( + Duration::from_millis(500), + test.client.put_cas(11, 1, b"recovered", b"retained", 0), + ) + .await; + test.stop().await; + assert!( + matches!(result, Ok(Ok(_))), + "stale first seed retained rejected route: {result:?}" + ); +} + +#[tokio::test] +async fn rejecting_stale_routes_does_not_remove_the_retry_budget() { + let test = TestDiscovery::start(false, false).await; + let result = test.client.put_cas(11, 1, b"bounded", b"value", 0).await; + test.stop().await; + assert!( + matches!(result, Err(Error::RetriesExhausted { attempts: 3, .. })), + "{result:?}" + ); +} + +#[tokio::test] +async fn concurrent_route_failures_share_the_replacement_discovery() { + let test = TestDiscovery::start(true, true).await; + let result = tokio::time::timeout(Duration::from_millis(500), async { + let mut requests = tokio::task::JoinSet::new(); + for id in 0u8..32 { + let client = std::sync::Arc::clone(&test.client); + requests.spawn(async move { client.put(11, 1, &[id], b"retained", None).await }); + } + let mut results = Vec::new(); + while let Some(result) = requests.join_next().await { + results.push(result.unwrap()); + } + results + }) + .await; + let fetches = test.fetches.load(std::sync::atomic::Ordering::Relaxed); + test.stop().await; + assert!( + matches!(result, Ok(ref results) if results.iter().all(Result::is_ok)), + "{result:?}" + ); + assert_eq!( + fetches, 2, + "the stale and replacement seeds should each be fetched once" + ); +} diff --git a/lib/crowdb-kv/Cargo.toml b/lib/crowdb-kv/Cargo.toml index 77550aadc..d6fa5a6ec 100644 --- a/lib/crowdb-kv/Cargo.toml +++ b/lib/crowdb-kv/Cargo.toml @@ -21,7 +21,7 @@ workspace = true test-util = [] [dependencies] -tokio = { workspace = true, features = ["rt", "macros", "time", "net", "fs", "sync", "io-util"] } +tokio = { workspace = true, features = ["rt", "rt-multi-thread", "macros", "time", "net", "fs", "sync", "io-util"] } tokio-stream = "0.1" tokio-util = { version = "0.7", features = ["rt"] } bytes = "1" @@ -106,6 +106,14 @@ path = "tests/store_test.rs" name = "wal_test" path = "tests/wal_test.rs" +[[test]] +name = "replay_memory_test" +path = "tests/replay_memory_test.rs" + +[[test]] +name = "replay_compaction_memory_test" +path = "tests/replay_compaction_memory_test.rs" + [[bench]] name = "wal" harness = false diff --git a/lib/crowdb-kv/src/cluster/group.rs b/lib/crowdb-kv/src/cluster/group.rs index 1bb131239..3bd786527 100644 --- a/lib/crowdb-kv/src/cluster/group.rs +++ b/lib/crowdb-kv/src/cluster/group.rs @@ -939,16 +939,13 @@ impl PxGroup { } /// R65: Leader-side `FetchGap` handler. Looks up the requested slot - /// in the local acceptor. If the leader has an accepted value, replies - /// with it (payload + ballot + term). If the leader doesn't have the - /// value, returns `None` — the slot is not yet chosen from this - /// leader's perspective, and the follower should retry later (the - /// leader's `repair_once` will eventually fill it, or a newer - /// `ChosenNotice` will re-notify). + /// in the local acceptor, restricted to the recovered leader's proven + /// contiguous chosen prefix. An accepted value alone is insufficient + /// evidence for a follower to learn it. pub(crate) fn handle_fetch_gap(&self, slot: SlotIndex) -> Option { let replica = &self.local_replica; - if !replica.is_leader() { - debug!(slot, role = ?replica.role(), leader = ?replica.believed_leader_id(), "handle_fetch_gap: not leader, ignoring"); + if !replica.is_leader() || !self.leader_read_ready() || slot > replica.contiguous_chosen() { + debug!(slot, role = ?replica.role(), leader = ?replica.believed_leader_id(), "handle_fetch_gap: recovered chosen proof unavailable"); return None; } let entry = replica.acceptor.accepted_at(slot)?; diff --git a/lib/crowdb-kv/src/cluster/group_election.rs b/lib/crowdb-kv/src/cluster/group_election.rs index f208dd947..7755a1821 100644 --- a/lib/crowdb-kv/src/cluster/group_election.rs +++ b/lib/crowdb-kv/src/cluster/group_election.rs @@ -23,14 +23,16 @@ use std::time::Duration; use tokio::task::JoinHandle; use tokio::time::Instant; use tokio_util::sync::CancellationToken; -use tracing::{debug, info, trace, warn, Instrument}; +use tracing::{debug, info, trace, Instrument}; -use crate::cluster::group::{AcceptAttempt, PrepareAttempt, PxGroup, RemoteReplicaKind}; +use crate::cluster::group::{PxGroup, RemoteReplicaKind}; use crate::cluster::local_replica::PxLocalReplicaRole; use crate::common::config::PxElectionConfig; use crate::paxos::roles::SlotIndex; use crate::paxos::PxNodeId; +mod recovery; + /// Bundle handed off from `run_candidate_election` to `run_leader_state` /// when a candidate wins quorum. Carries the floor / ceiling needed by /// [`LeaderElection::run_bulk_phase1`] under the new tenure's cancel @@ -84,9 +86,8 @@ pub trait LeaderElection { /// /// Inputs: /// - /// - `floor` = `max(local.contiguous_chosen, peer_contiguous_chosen_max)` - /// — values from peer `RequestVote` / `PreVote` replies (the election - /// driver supplies the aggregate via `peer_contiguous_chosen_max`). + /// - `floor` = the local contiguous-chosen frontier. A peer's higher + /// frontier does not prove that its payloads are present locally. /// - `ceiling` = `max(local.acceptor.highest_seen_slot, /// self.next_slot - 1, /// peer_highest_seen_slot_max)`. @@ -100,9 +101,9 @@ pub trait LeaderElection { /// is decided (and the contiguous-chosen watermark can advance). /// 3. Re-Accept via the existing `run_accept_phase`. /// - /// After issuing (not waiting on completion of) the batch, `next_slot` - /// is bumped to `ceiling + 1` so steady-state proposals continue past - /// the repaired range (§4.4). + /// At most `cfg.bulk_prepare_window` slots recover concurrently. Once + /// the entire range is resolved and applied, reads may be admitted. + /// `next_slot` is advanced to `ceiling + 1` after the sweep. /// /// Cancellation: the `cancel` token is checked before each slot. On /// cancel the loop aborts without re-Accepting any further slots — the @@ -176,8 +177,6 @@ impl LeaderElection for PxGroup { *driver_guard = Some(handle); } - #[tracing::instrument(level = "info", skip_all, fields(s = self.log_store_id().unwrap_or(0), g = self.group_id, replica = self.local_replica().id, term))] - #[allow(clippy::too_many_lines)] async fn run_bulk_phase1( &self, term: u64, @@ -186,146 +185,14 @@ impl LeaderElection for PxGroup { cfg: PxElectionConfig, cancel: CancellationToken, ) { - let replica = self.local_replica(); - let group_id = self.group_id; - // Voting-only quorum -- see the matching fix/comment on - // `PxGroup::propose` in `group.rs`. - let quorum = self.quorum(); - - // Floor is the leader's OWN committed frontier — deliberately NOT maxed - // with `peer_contiguous_chosen_max`. After a restart a replica can win - // election while missing committed slots it never received (the value - // lived only as a `ChosenNotice` watermark, which carries no payload). - // Maxing the floor with the peers' commit point would SKIP exactly - // those slots, leaving the new leader serving a stale/old value for a - // committed key (e.g. a resurrected delete, or a missing put). Sweeping - // from the leader's own frontier forces Phase 1 to re-derive every - // higher slot from the quorum and adopt the real chosen value. This is - // safe because Phase 1 contacts a quorum: a committed value is held by - // at least one quorum member, so it is recovered rather than - // overwritten. - let _ = peer_contiguous_chosen_max; - let floor = replica.contiguous_chosen(); - let local_ceiling = replica.highest_seen_slot(); - let next_slot_minus_one = self.next_slot.load(Ordering::Acquire).saturating_sub(1); - let ceiling = local_ceiling - .max(next_slot_minus_one) - .max(peer_highest_seen_slot_max); - - debug!( - group_id, + self.recover_open_prefix( term, - floor, - ceiling, - local_contiguous_chosen = replica.contiguous_chosen(), - local_highest_seen_slot = local_ceiling, peer_contiguous_chosen_max, peer_highest_seen_slot_max, - "bulk phase 1 start" - ); - - if ceiling <= floor { - tokio::select! { - () = replica.await_apply_fence(floor) => {} - () = cancel.cancelled() => return, - } - self.leader_read_ready.store(true, Ordering::Release); - debug!(term, "bulk phase 1 skipped (empty range)"); - return; - } - - let mut slots_repaired = 0u64; - let mut all_resolved = true; - let window = cfg.bulk_prepare_window.max(1); - - for slot in (floor + 1)..=ceiling { - if cancel.is_cancelled() { - warn!( - group_id, - term, slot, slots_repaired, "bulk phase 1 cancelled (step down)" - ); - return; - } - if slots_repaired >= window { - tokio::task::yield_now().await; - slots_repaired = 0; - } - - // Issue Phase-1 Prepare at ballot (0, me) under term T with an - // empty payload so any adopted foreign value comes strictly - // from a remote's previously-Accepted entry; if none exist - // the entry stays a NoOp (we re-tag below). - let attempt = self - .run_prepare_phase(replica, slot, bytes::Bytes::new(), quorum, 0) - .await; - let mut entry = match attempt { - PrepareAttempt::Proceed { entry, .. } => entry, - PrepareAttempt::Retry { error, .. } | PrepareAttempt::Fail { error } => { - all_resolved = false; - warn!( - group_id, - term, - slot, - error = error.keyword(), - "bulk phase 1 prepare failed; will be retried by next leader" - ); - continue; - } - }; - - entry.term = term; - - match self.run_accept_phase(replica, &entry, quorum).await { - AcceptAttempt::Chosen => { - replica.learn_chosen(&entry, &[]).await; - self.fan_out_chosen_notice(&entry, group_id); - slots_repaired += 1; - } - AcceptAttempt::Retry { error, .. } | AcceptAttempt::Fail { error } => { - all_resolved = false; - warn!( - group_id, - term, - slot, - error = error.keyword(), - "bulk phase 1 accept failed; will be retried by next leader" - ); - } - } - } - - let next = ceiling.saturating_add(1); - self.next_slot.fetch_max(next, Ordering::AcqRel); - if !all_resolved { - warn!( - group_id, - term, ceiling, "bulk phase 1 incomplete; leader recovery remains closed" - ); - return; - } - tokio::select! { - () = replica.await_apply_fence(ceiling) => {} - () = cancel.cancelled() => return, - } - self.leader_read_ready.store(true, Ordering::Release); - // R65: advance `known_commit_slot` to the leader's own - // `contiguous_chosen` after the sweep. Leaders don't receive - // heartbeats or ChosenNotice, so without this the apply loop - // would never learn about slots the leader accepted as a follower - // (before winning the election) or slots resolved by the sweep. - // All slots up to `contiguous_chosen` are now confirmed chosen by - // the quorum that responded to Prepare — safe to apply. - let cc = replica.contiguous_chosen(); - replica.advance_known_commit_slot(cc); - replica.wake_apply_loop(); - debug!( - group_id, - term, - ceiling, - next_slot = next, - contiguous_chosen = cc, - "bulk phase 1 done" - ); + cfg, + cancel, + ) + .await; } } diff --git a/lib/crowdb-kv/src/cluster/group_election/recovery.rs b/lib/crowdb-kv/src/cluster/group_election/recovery.rs new file mode 100644 index 000000000..bcd3b56a8 --- /dev/null +++ b/lib/crowdb-kv/src/cluster/group_election/recovery.rs @@ -0,0 +1,113 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Bounded concurrent recovery of a new leader's open prefix. + +use crate::{ + cluster::group::{AcceptAttempt, PrepareAttempt, PxGroup}, + common::config::PxElectionConfig, +}; +use futures::{stream, StreamExt}; +use std::sync::atomic::Ordering; +use tokio_util::sync::CancellationToken; +use tracing::{debug, warn}; + +impl PxGroup { + #[tracing::instrument(level = "info", skip_all, fields(s = self.log_store_id().unwrap_or(0), g = self.group_id, replica = self.local_replica().id, term))] + pub(super) async fn recover_open_prefix( + &self, + term: u64, + peer_floor: u64, + peer_ceiling: u64, + cfg: PxElectionConfig, + cancel: CancellationToken, + ) { + let replica = self.local_replica(); + // Peers' committed frontiers carry no payload. Start at our own + // frontier so quorum Phase 1 recovers every missing chosen value. + let floor = replica.contiguous_chosen(); + let ceiling = replica + .highest_seen_slot() + .max(self.next_slot.load(Ordering::Acquire).saturating_sub(1)) + .max(peer_ceiling); + debug!(term, floor, ceiling, peer_floor, "bulk phase 1 start"); + if ceiling > floor { + let window = usize::try_from(cfg.bulk_prepare_window.max(1)).unwrap_or(usize::MAX); + let mut slots = stream::iter((floor + 1)..=ceiling) + .map(|slot| self.recover_prefix_slot(term, slot, &cancel)) + .buffer_unordered(window); + let mut resolved = true; + loop { + tokio::select! { + biased; + () = cancel.cancelled() => return, + result = slots.next() => match result { + Some(success) => resolved &= success, + None => break, + }, + } + } + self.next_slot + .fetch_max(ceiling.saturating_add(1), Ordering::AcqRel); + if !resolved { + warn!( + term, + ceiling, "bulk phase 1 incomplete; leader recovery remains closed" + ); + return; + } + } + tokio::select! { + biased; + () = cancel.cancelled() => return, + () = replica.await_apply_fence(ceiling.max(floor)) => {} + } + if replica.current_term_snapshot() != term || !replica.is_leader() { + return; + } + self.leader_read_ready.store(true, Ordering::Release); + replica.advance_known_commit_slot(replica.contiguous_chosen()); + replica.wake_apply_loop(); + debug!(term, ceiling, "bulk phase 1 done"); + } + + async fn recover_prefix_slot(&self, term: u64, slot: u64, cancel: &CancellationToken) -> bool { + let replica = self.local_replica(); + if cancel.is_cancelled() || replica.current_term_snapshot() != term || !replica.is_leader() { + return false; + } + let mut entry = match self + .run_prepare_phase(replica, slot, bytes::Bytes::new(), self.quorum(), 0) + .await + { + PrepareAttempt::Proceed { entry, .. } => entry, + PrepareAttempt::Retry { error, .. } | PrepareAttempt::Fail { error } => { + warn!( + slot, + error = error.keyword(), + "bulk phase 1 prepare failed; leader recovery remains closed" + ); + return false; + } + }; + if cancel.is_cancelled() || replica.current_term_snapshot() != term || !replica.is_leader() { + return false; + } + entry.term = term; + match self.run_accept_phase(replica, &entry, self.quorum()).await { + AcceptAttempt::Chosen => { + replica.learn_chosen(&entry, &[]).await; + self.fan_out_chosen_notice(&entry, self.group_id); + true + } + AcceptAttempt::Retry { error, .. } | AcceptAttempt::Fail { error } => { + warn!( + slot, + error = error.keyword(), + "bulk phase 1 accept failed; leader recovery remains closed" + ); + false + } + } + } +} diff --git a/lib/crowdb-kv/src/cluster/group_election_leader.rs b/lib/crowdb-kv/src/cluster/group_election_leader.rs index a4d0afba7..f394333ec 100644 --- a/lib/crowdb-kv/src/cluster/group_election_leader.rs +++ b/lib/crowdb-kv/src/cluster/group_election_leader.rs @@ -503,8 +503,13 @@ impl PxGroup { } // Admin step-down via StepDown RPC. () = replica.admin_step_down_signal.notified() => { - self.step_down(&tenure_cancel, leader_term, StepDownReason::Admin); - return; + // Accepted requests fence the role before publishing the wakeup. + // A permit retained from an earlier tenure cannot dismiss a + // replica that has since become leader again. + if replica.role() != PxLocalReplicaRole::Leader { + self.step_down(&tenure_cancel, leader_term, StepDownReason::Admin); + return; + } } _ = ticker.tick() => { match self.run_heartbeat_round(cfg, leader_term).await { @@ -522,7 +527,10 @@ impl PxGroup { } // Keep at most one repair in flight. Prepare or // persistence IO must not occupy the heartbeat loop. - if gap_repair.as_ref().map_or(true, tokio::task::JoinHandle::is_finished) { + if self.leader_read_ready() + && recovery_task.as_ref().map_or(true, tokio::task::JoinHandle::is_finished) + && gap_repair.as_ref().map_or(true, tokio::task::JoinHandle::is_finished) + { gap_repair = self.start_gap_repair(&tenure_cancel); } } diff --git a/lib/crowdb-kv/src/cluster/group_fetchgap.rs b/lib/crowdb-kv/src/cluster/group_fetchgap.rs index cbe3b850b..068e345d5 100644 --- a/lib/crowdb-kv/src/cluster/group_fetchgap.rs +++ b/lib/crowdb-kv/src/cluster/group_fetchgap.rs @@ -4,17 +4,26 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::cast_possible_truncation)] -use std::sync::atomic::Ordering; -use std::sync::Weak; +use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, Weak, +}; use std::time::Duration; use tokio_util::sync::CancellationToken; -use tracing::{debug, warn}; +use tracing::debug; use crate::cluster::group::PxGroup; -use crate::paxos::roles::Acceptor; use crate::paxos::PxGroupId; +struct FetchGapInFlight(Arc); + +impl Drop for FetchGapInFlight { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::AcqRel); + } +} + /// R65: Follower-side `FetchGap` driver. Periodically checks the local /// replica's `gap_slots` set and sends `FetchGap` to the leader for each /// gap. On reply, overwrites the stale/missing value and wakes the apply @@ -25,6 +34,7 @@ pub(crate) async fn run_fetchgap_driver( cancel: CancellationToken, ) { let tick = Duration::from_millis(10); + let mut large_backlog = false; loop { if cancel.is_cancelled() { return; @@ -49,21 +59,17 @@ pub(crate) async fn run_fetchgap_driver( let Some(remote) = group.get_remote_replica(leader_id) else { continue; }; - // R65: snapshot fallback — if gap count exceeds the threshold, - // skip FetchGap and log a warning. A full automatic snapshot - // install for running replicas is a follow-up; for now the - // follower will eventually catch up via ChosenNotice + heartbeat. + // Running replicas have no atomic snapshot replacement yet. Keep + // draining large backlogs with the existing fixed inflight bound. let gap_count = group.local_replica().gap_slots.lock().len(); let threshold = group.config.election.catchup_snapshot_threshold; - if gap_count > threshold as usize { - warn!( + if gap_count > threshold as usize && !large_backlog { + debug!( group_id, - gap_count, - threshold, - "gap count exceeds snapshot threshold; FetchGap skipped (snapshot fallback not yet automatic)" + gap_count, threshold, "snapshot catch-up is unavailable; continuing bounded FetchGap" ); - continue; } + large_backlog = gap_count > threshold as usize; // Clone the crowdb-rpc transport + endpoint so the spawned task // can call `transport.send_fetch_gap()` directly. let rpc_transport = remote.rpc_transport().cloned(); @@ -81,6 +87,7 @@ pub(crate) async fn run_fetchgap_driver( let gap_slots = replica.gap_slots.clone(); let learner = replica.learner.clone(); let acceptor = replica.acceptor.clone(); + let wal = replica.wal().cloned(); let apply_notify = replica.apply_notify.clone(); let fetchgap_received = replica .replication_handles @@ -95,16 +102,17 @@ pub(crate) async fn run_fetchgap_driver( let rpc_transport_clone = rpc_transport.clone(); let rpc_endpoint_clone = rpc_endpoint.clone(); let rpc_timeout_clone = rpc_timeout; - let fetchgap_inflight_clone = fetchgap_inflight.clone(); + let inflight = FetchGapInFlight(fetchgap_inflight.clone()); let gap_slots_clone = gap_slots.clone(); let learner_clone = learner.clone(); let acceptor_clone = acceptor.clone(); + let wal_clone = wal.clone(); let apply_notify_clone = apply_notify.clone(); let fetchgap_received_clone = fetchgap_received.clone(); let fetchgap_failed_clone = fetchgap_failed.clone(); tokio::spawn(async move { + let _inflight = inflight; let Some(ref transport) = rpc_transport_clone else { - fetchgap_inflight_clone.fetch_sub(1, Ordering::AcqRel); if let Some(c) = &fetchgap_failed_clone { c.inc(); } @@ -132,7 +140,6 @@ pub(crate) async fn run_fetchgap_driver( )) }) .and_then(|r| r); - fetchgap_inflight_clone.fetch_sub(1, Ordering::AcqRel); match result { Ok(resp) => { if let Some(c) = &fetchgap_received_clone { @@ -153,7 +160,30 @@ pub(crate) async fn run_fetchgap_driver( term: resp.term, payload: resp.payload, }; - acceptor_clone.accept(&entry).await; + // Historical chosen values may belong to an older + // election term. Persist their original entry before + // exposing an applied frontier or snapshot coverage. + let persisted = if let Some(wal) = &wal_clone { + match wal + .append(&crate::wal::record::WALRecord::from_accepted(group_id, &entry)) + .await + { + Ok(_) => true, + Err(error) => { + tracing::error!(group_id, slot, %error, "FetchGap persistence failed; retaining gap for later recovery"); + false + } + } + } else { + true + }; + if !persisted || !acceptor_clone.install_chosen(&entry).await { + gap_slots_clone.lock().insert(slot); + if let Some(c) = &fetchgap_failed_clone { + c.inc(); + } + return; + } learner_clone.update_chosen_frontier(resp.slot, resp.term); apply_notify_clone.notify_one(); } diff --git a/lib/crowdb-kv/src/cluster/group_tests.rs b/lib/crowdb-kv/src/cluster/group_tests.rs index f57e9e1f8..31cdbe8e4 100644 --- a/lib/crowdb-kv/src/cluster/group_tests.rs +++ b/lib/crowdb-kv/src/cluster/group_tests.rs @@ -14,6 +14,14 @@ use crate::paxos::PxNodeId; /// repair step, and peer-applied injection — to integration tests under /// `tests/` without permanently widening the production public API. impl PxGroup { + /// Observe a missing chosen value without delivering its Accepted frame. + pub fn note_missing_chosen_for_tests(&self, slot: SlotIndex) { + let replica = self.local_replica(); + replica.record_gap(slot); + replica.note_chosen(slot, replica.current_term_snapshot()); + replica.wake_apply_loop(); + } + /// Whether a conditional mutation currently owns one key's admission. #[must_use] pub fn conditional_write_pending_for_tests(&self, key: &[u8]) -> bool { diff --git a/lib/crowdb-kv/src/cluster/local_replica_election.rs b/lib/crowdb-kv/src/cluster/local_replica_election.rs index edb113113..65e5a8a2d 100644 --- a/lib/crowdb-kv/src/cluster/local_replica_election.rs +++ b/lib/crowdb-kv/src/cluster/local_replica_election.rs @@ -127,12 +127,16 @@ impl PxLocalReplica { /// - `req.term > current_term` (we'd vote in that term), AND /// - candidate's log is up-to-date, AND /// - `now >= vote_lockout_until`. + /// + /// A leader's quorum-renewed lease also protects its own vote after the + /// original self-vote lockout has expired. pub(super) fn handle_pre_vote(&self, req: VoteRequestPayload) -> VoteReply { let now = Instant::now(); let state = self.election_state.lock(); let granted = req.term > state.current_term && self.candidate_log_up_to_date(&req) - && now >= state.vote_lockout_until; + && now >= state.vote_lockout_until + && (state.role != PxLocalReplicaRole::Leader || self.lease_read_until() <= now); let term = state.current_term; drop(state); let (contiguous_chosen, last_chosen_term, highest_seen_slot) = self.frontier_triple(); @@ -167,7 +171,8 @@ impl PxLocalReplica { let (granted, term) = { let mut state = self.election_state.lock(); - let lockout_ok = now >= state.vote_lockout_until; + let lockout_ok = now >= state.vote_lockout_until + && (state.role != PxLocalReplicaRole::Leader || self.lease_read_until() <= now); let term_ok = req.term > state.current_term || (req.term == state.current_term && state.voted_for.map_or(true, |v| v == req.candidate_id)); diff --git a/lib/crowdb-kv/src/cluster/local_replica_replay.rs b/lib/crowdb-kv/src/cluster/local_replica_replay.rs index 797318ce9..74eec8dfc 100644 --- a/lib/crowdb-kv/src/cluster/local_replica_replay.rs +++ b/lib/crowdb-kv/src/cluster/local_replica_replay.rs @@ -63,22 +63,25 @@ impl PxLocalReplica { let resume_from = engine.resume_from_slot(); let replica = Self::new_with_learner(id, role, PxLearner::with_engine(engine)); - // Pass 1: rebuild acceptor (promise + accept) state from the WAL. + // Accepted history and later promises are independent durable facts. + // FetchGap may learn an old chosen value after a higher promise; replay + // must retain that value and the higher fence regardless of record order. + for record in replay + .records + .iter() + .filter(|record| record.record_type == crate::wal::record::RecordType::Accepted) + { + let entry = record.to_log_entry().ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "restore replay accepted missing log entry", + ) + })?; + let _ = replica.acceptor.accept(&entry).await; + } for record in &replay.records { - match record.record_type { - crate::wal::record::RecordType::Promised => { - let _ = replica.acceptor.prepare(record.slot, record.ballot).await; - } - crate::wal::record::RecordType::Accepted => { - let entry = record.to_log_entry().ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "restore replay accepted missing log entry", - ) - })?; - let _ = replica.acceptor.accept(&entry).await; - } - crate::wal::record::RecordType::VoteGranted => {} + if record.record_type == crate::wal::record::RecordType::Promised { + let _ = replica.acceptor.prepare(record.slot, record.ballot).await; } } @@ -124,25 +127,19 @@ impl PxLocalReplica { // legitimately touches. There is no safe way to "fall back" to // replaying it once the engine is past that floor. // - // Seed the frontier to `(resume_from, term-at-resume_from)` via - // `seed_resume_frontier` when the just-rebuilt acceptor has an - // accepted entry at that exact slot (the expected case: an engine - // can only ever have durably applied a slot that was itself - // accepted and WAL-logged, and Pass 1 rebuilds the *entire* WAL - // history). If it's missing (e.g. a WAL segment lost/GC'd after the - // engine already durably flushed that slot -- not expected, but not - // an invariant this restore path should trust blindly), leave the - // frontier at the fresh learner's default (`0`) rather than guess a - // term: under-reporting `contiguous_chosen`/`last_chosen_term` only - // costs more conservative heartbeat catch-up / safe-read bounds, - // never incorrectness, unlike attempting the skipped replay. + // A recovered engine proves the complete durable prefix even when + // WAL reclamation removed its boundary record. Preserve that prefix; + // requesting it again can never succeed once peers reclaimed it too. + // Term zero explicitly records unknown historical term. Election + // freshness uses the durable acceptor tip, not this chosen watermark. let highest = replica.acceptor.highest_seen_slot(); - let resume_from = resume_from.min(highest); let start_slot = if resume_from > 0 { - if let Some(entry) = replica.acceptor.accepted_at(resume_from) { - replica.learner.seed_resume_frontier(resume_from, entry.term); - } - resume_from + 1 + let term = replica + .acceptor + .accepted_at(resume_from) + .map_or(0, |entry| entry.term); + replica.learner.seed_resume_frontier(resume_from, term); + resume_from.saturating_add(1) } else { 1 }; diff --git a/lib/crowdb-kv/src/cluster/px_kv_store.rs b/lib/crowdb-kv/src/cluster/px_kv_store.rs index b4520effa..fc5fb779d 100644 --- a/lib/crowdb-kv/src/cluster/px_kv_store.rs +++ b/lib/crowdb-kv/src/cluster/px_kv_store.rs @@ -660,11 +660,15 @@ impl PxKvStore { } pub(crate) fn missing_group_response(request_id: u64, request_create_ms: u64) -> crate::rpc::KvResponse { - crate::rpc::KvResponse::err( + let mut response = crate::rpc::KvResponse::err( "no kv group configured for request group_id".to_string(), request_id, request_create_ms, - ) + ); + // A cached route may name a replica removed from this process. The caller + // must discover the current leader rather than repeat the obsolete route. + response.error_code = crate::rpc::KvErrorCode::KvErrorNotLeader as i32; + response } /// Build a failed [`crate::rpc::KvScanResponse`] carrying `error` and an diff --git a/lib/crowdb-kv/src/cluster/px_kv_store_handler.rs b/lib/crowdb-kv/src/cluster/px_kv_store_handler.rs index 208ee5072..acb5d93c5 100644 --- a/lib/crowdb-kv/src/cluster/px_kv_store_handler.rs +++ b/lib/crowdb-kv/src/cluster/px_kv_store_handler.rs @@ -237,12 +237,14 @@ impl KvStore for PxKvStore { request_create_ms: u64, ) -> crate::rpc::KvScanResponse { let Some(operations) = self.group_operations(group_id) else { - return scan_err( + let mut response = scan_err( format!("group {group_id} not found in store {}", self.store_id), String::new(), request_id, request_create_ms, ); + response.error_code = crate::rpc::KvErrorCode::KvErrorNotLeader as i32; + return response; }; let request = KvGroupScanRequest { prefix: Bytes::copy_from_slice(prefix), @@ -516,13 +518,15 @@ impl KvStore for PxKvStore { request_create_ms: u64, ) -> crate::rpc::KvJournalScanResponse { let Some(group) = self.get_group(group_id) else { - return journal_scan_err( + let mut response = journal_scan_err( format!("group {group_id} not found in store {}", self.store_id), String::new(), false, request_id, request_create_ms, ); + response.error_code = crate::rpc::KvErrorCode::KvErrorNotLeader as i32; + return response; }; // Read-mode routing mirrors `kv_scan`: linearizable runs the @@ -568,7 +572,8 @@ impl KvStore for PxKvStore { } else { max_slot.min(read_slot) }; - if effective_max < min_slot { + let first_slot = min_slot.max(1); + if effective_max < first_slot { // Empty range — return an empty success. return crate::rpc::KvJournalScanResponse { version: 1, @@ -592,7 +597,22 @@ impl KvStore for PxKvStore { // `end_slot_exclusive` = effective_max + 1 (effective_max is // inclusive per the proto contract). let end_exclusive = effective_max.saturating_add(1); - let entries = acceptor.accepted_iter_range(min_slot, end_exclusive); + let entries = acceptor.accepted_iter_range(first_slot, end_exclusive); + // Restart can restore the applied prefix from the durable engine while + // its earlier WAL segments are already gone. The live trim watermark + // has not necessarily been restored or advanced by maintenance yet. + // Also cover reclamation racing the trim check above: copies of every + // applied slot, including NoOps, must exist before filtering by key. + let expected_entries = effective_max - first_slot + 1; + if u64::try_from(entries.len()).unwrap_or(u64::MAX) != expected_entries { + return journal_scan_err( + format!("journal_scan history is incomplete in slots {first_slot}..={effective_max}"), + String::new(), + true, + request_id, + request_create_ms, + ); + } // Decode each slot's payload batch and emit matching ops in // slot order (within a slot, ops are in batch order). diff --git a/lib/crowdb-kv/src/common/config.rs b/lib/crowdb-kv/src/common/config.rs index e918d1d55..064475d1b 100644 --- a/lib/crowdb-kv/src/common/config.rs +++ b/lib/crowdb-kv/src/common/config.rs @@ -259,12 +259,11 @@ pub struct PxElectionConfig { /// Maximum admissible clock skew across the cluster. Subtracted from the /// leader's `lease_read_until` to remain safe under skew. pub max_clock_skew_ms: u64, - /// Slots scanned per bulk-Phase-1 batch (new-leader open-prefix repair). + /// Maximum concurrent slots in new-leader open-prefix recovery. pub bulk_prepare_window: u64, - /// R65: gap count threshold for snapshot fallback. When a follower's - /// gap count exceeds this, `FetchGap` is skipped and a warning is logged - /// (full automatic snapshot install for running replicas is a - /// follow-up). Default matches `bulk_prepare_window` (1024). + /// Gap count at which snapshot catch-up would be useful. Atomic snapshot + /// replacement for running replicas is not implemented, so bounded + /// `FetchGap` continues above this diagnostic threshold. Default: 1024. pub catchup_snapshot_threshold: u64, /// Test-only override: when `true`, the election driver task is not /// spawned. Used by `testkit::cluster::start_cluster` to keep legacy M1/M2 diff --git a/lib/crowdb-kv/src/paxos/acceptor.rs b/lib/crowdb-kv/src/paxos/acceptor.rs index 96c3f9616..54eb9df54 100644 --- a/lib/crowdb-kv/src/paxos/acceptor.rs +++ b/lib/crowdb-kv/src/paxos/acceptor.rs @@ -110,6 +110,29 @@ impl PxAcceptor { self.slots.len() } + /// Retain a value proved chosen by the current recovered leader without + /// answering Accept or weakening a later promise. A higher accepted ballot + /// may already carry the same chosen value; conflicting evidence fails closed. + pub(crate) async fn install_chosen(&self, entry: &PxLogEntry) -> bool { + let _ = self.prepare(entry.slot, entry.ballot).await; + let Some(guard) = self.open_slot(entry.slot) else { + return false; + }; + let node = guard.value(); + self.bump_highest_seen(entry.slot); + loop { + let previous = node.accepted.load(Ordering::Acquire); + if let Some(accepted) = node.accepted() { + if accepted.ballot >= entry.ballot { + return accepted.payload == entry.payload; + } + } + if node.cas_accepted(previous, entry.clone()).is_ok() { + return true; + } + } + } + fn open_slot(&self, slot: SlotIndex) -> Option> { if slot < self.trim_slot() { return None; diff --git a/lib/crowdb-kv/src/wal/block_backend.rs b/lib/crowdb-kv/src/wal/block_backend.rs index bb795b1d6..1ada1f040 100644 --- a/lib/crowdb-kv/src/wal/block_backend.rs +++ b/lib/crowdb-kv/src/wal/block_backend.rs @@ -13,6 +13,20 @@ use parking_lot::Mutex; use super::io_backend::OpenOptions; use super::pipeline_backend::WalBlockAlignment; +/// Hand the multi-thread runtime's worker duties back before blocking on storage. +/// Borrowed WAL buffers stay on the caller's stack without an extra payload copy. +/// Current-thread runtimes retain their synchronous behavior: they cannot hand off +/// a worker and Tokio rejects `block_in_place` there. +pub(super) fn blocking_io(operation: impl FnOnce() -> T) -> T { + let can_handoff = tokio::runtime::Handle::try_current() + .is_ok_and(|handle| handle.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread); + if can_handoff { + tokio::task::block_in_place(operation) + } else { + operation() + } +} + #[derive(Clone, Default)] pub struct BlockDeviceController { inner: Arc, @@ -103,6 +117,16 @@ pub struct BlockDevice { #[cfg_attr(not(target_os = "linux"), allow(dead_code))] use_direct_io: bool, rmw_count: Arc, + #[cfg(feature = "test-util")] + io_gate: Arc>, +} + +#[cfg(feature = "test-util")] +struct TestIoGate { + sync: bool, + armed: AtomicBool, + entered: Arc, + release: Arc, } impl BlockDevice { @@ -134,6 +158,40 @@ impl BlockDevice { alignment, use_direct_io, rmw_count: Arc::new(AtomicU64::new(0)), + #[cfg(feature = "test-util")] + io_gate: Arc::new(std::sync::OnceLock::new()), + } + } + + /// Hold the next physical write or sync before entering the real file operation. + /// + /// # Panics + /// Panics if a gate was already installed on this device. + #[cfg(feature = "test-util")] + pub fn set_io_gate_for_tests( + &self, + sync: bool, + entered: Arc, + release: Arc, + ) { + assert!(self + .io_gate + .set(TestIoGate { + sync, + armed: AtomicBool::new(true), + entered, + release + }) + .is_ok()); + } + + #[cfg(feature = "test-util")] + fn pause_io_for_tests(&self, sync: bool) { + if let Some(gate) = self.io_gate.get() { + if gate.sync == sync && gate.armed.swap(false, Ordering::AcqRel) { + gate.entered.wait(); + gate.release.wait(); + } } } @@ -159,6 +217,10 @@ impl BlockDevice { } pub(crate) fn open_segment(&self, segment_path: &Path, opts: &OpenOptions) -> io::Result { + blocking_io(|| self.open_segment_file(segment_path, opts)) + } + + fn open_segment_file(&self, segment_path: &Path, opts: &OpenOptions) -> io::Result { let segment_path = segment_path.to_path_buf(); let mut std_opts = std::fs::OpenOptions::new(); std_opts.read(true).write(true); @@ -185,32 +247,34 @@ impl BlockDevice { #[allow(clippy::unused_self)] pub(super) fn rename_segment(&self, from: &Path, to: &Path) -> io::Result<()> { - std::fs::rename(from, to) + blocking_io(|| std::fs::rename(from, to)) } #[allow(clippy::unused_self)] pub(super) fn unlink_segment(&self, segment_path: &Path) -> io::Result<()> { - std::fs::remove_file(segment_path) + blocking_io(|| std::fs::remove_file(segment_path)) } #[allow(clippy::unused_self)] pub(super) fn list_layout(&self, layout_path: &Path) -> io::Result> { - let mut entries = Vec::new(); - let rd = std::fs::read_dir(layout_path)?; - for entry in rd.flatten() { - entries.push(entry.path()); - } - Ok(entries) + blocking_io(|| { + let mut entries = Vec::new(); + let rd = std::fs::read_dir(layout_path)?; + for entry in rd.flatten() { + entries.push(entry.path()); + } + Ok(entries) + }) } #[allow(clippy::unused_self)] pub(super) fn create_layout(&self, layout_path: &Path) -> io::Result<()> { - std::fs::create_dir_all(layout_path) + blocking_io(|| std::fs::create_dir_all(layout_path)) } #[allow(clippy::unused_self)] pub(super) fn contains_path(&self, path: &Path) -> bool { - path.exists() + blocking_io(|| path.exists()) } } @@ -227,12 +291,16 @@ pub(crate) struct BlockSegment { impl BlockSegment { pub(super) fn write_at(&self, data: &[u8], offset: u64) -> io::Result { + #[cfg(feature = "test-util")] + self.device.pause_io_for_tests(false); self.write_bytes_to_file(&self.file, data, offset)?; self.device.write_count.fetch_add(1, Ordering::AcqRel); Ok(data.len()) } pub(super) fn write_vectored_at(&self, bufs: &[std::io::IoSlice<'_>], offset: u64) -> io::Result { + #[cfg(feature = "test-util")] + self.device.pause_io_for_tests(false); let total_len: usize = bufs.iter().map(|b| b.len()).sum(); let mut cur_offset = offset; for buf in bufs { @@ -312,6 +380,8 @@ impl BlockSegment { } pub(super) fn fdatasync(&self) -> io::Result<()> { + #[cfg(feature = "test-util")] + self.device.pause_io_for_tests(true); self.device.fdatasync_count.fetch_add(1, Ordering::AcqRel); self.file.sync_data()?; Ok(()) diff --git a/lib/crowdb-kv/src/wal/gc.rs b/lib/crowdb-kv/src/wal/gc.rs index bdd81a167..74ae7d980 100644 --- a/lib/crowdb-kv/src/wal/gc.rs +++ b/lib/crowdb-kv/src/wal/gc.rs @@ -96,12 +96,22 @@ pub async fn run_gc_with_watermark(wal: &WalEngine, gc_slot: u64) -> io::Result< let mut unlinked = 0usize; let disk_paths = wal.disk_group_paths(); - // Collect segments eligible for GC. - let eligible: Vec<(u64, usize, std::path::PathBuf)> = wal - .index() - .segments_snapshot() + // Slot progress cannot replace election durability. Keep one segment proving + // the highest term and one carrying the latest vote, including mixed segments. + let segments = wal.index().segments_snapshot(); + let term_segment = segments + .iter() + .max_by_key(|meta| (meta.max_term, meta.segment_id)) + .map(|meta| meta.segment_id); + let vote_segment = segments + .iter() + .filter_map(|meta| meta.vote_term.map(|term| (term, meta.segment_id))) + .max() + .map(|(_, id)| id); + let eligible: Vec<(u64, usize, std::path::PathBuf)> = segments .iter() .filter(|meta| meta.max_slot > 0 && meta.max_slot < gc_slot) + .filter(|meta| Some(meta.segment_id) != term_segment && Some(meta.segment_id) != vote_segment) .map(|meta| { let dir = &disk_paths[meta.disk_idx]; let filename = format!("seg-{:07}.ck", meta.segment_id); diff --git a/lib/crowdb-kv/src/wal/index.rs b/lib/crowdb-kv/src/wal/index.rs index d3d974aff..8a4ff7224 100644 --- a/lib/crowdb-kv/src/wal/index.rs +++ b/lib/crowdb-kv/src/wal/index.rs @@ -38,6 +38,10 @@ pub struct SegmentMeta { pub min_slot: SlotIndex, pub max_slot: SlotIndex, pub record_count: u32, + /// Highest election term observed in any verified record in this segment. + pub max_term: u64, + /// Highest term of a vote record; its segment must survive election-state GC. + pub vote_term: Option, } impl SegmentIndex { @@ -113,6 +117,37 @@ pub struct ShardedSegmentIndex { } impl ShardedSegmentIndex { + /// Seed the existing pipeline shards before any writer can publish new records. + /// + /// # Errors + /// Returns an error when recovered locations refer to an unavailable disk. + pub(super) fn from_replay( + pipeline_count: usize, + group_id: u64, + recovered: SegmentIndex, + ) -> std::io::Result { + let mut index = Self::new(pipeline_count, group_id); + for (slot, location) in recovered.map { + let shard = index.shards.get_mut(location.disk_idx).ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "replayed WAL disk is unavailable", + ) + })?; + shard.get_mut().insert(slot, location); + } + for meta in recovered.segments.into_values() { + let shard = index.shards.get_mut(meta.disk_idx).ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "replayed WAL disk is unavailable", + ) + })?; + shard.get_mut().register_segment(meta); + } + Ok(index) + } + /// Create one independent shard per WAL pipeline. /// /// # Panics diff --git a/lib/crowdb-kv/src/wal/pipeline_writer.rs b/lib/crowdb-kv/src/wal/pipeline_writer.rs index 3db5e102c..defa218cb 100644 --- a/lib/crowdb-kv/src/wal/pipeline_writer.rs +++ b/lib/crowdb-kv/src/wal/pipeline_writer.rs @@ -75,6 +75,8 @@ pub(crate) struct PendingWrite { pub encoded: EncodedRecord, /// Slot index for index update (0 = metadata record, not indexed). pub slot: SlotIndex, + pub term: u64, + pub is_vote: bool, /// Ack channel: resolves with the record's `SlotLocation` when the /// covering flush succeeds. pub ack: oneshot::Sender>, @@ -640,6 +642,10 @@ async fn write_batch( } } segment.record_count += 1; + segment.max_term = segment.max_term.max(req.term); + if req.is_vote { + segment.vote_term = Some(segment.vote_term.map_or(req.term, |term| term.max(req.term))); + } } // Single durable flush for the whole batch — skipped when skip_fsync is @@ -689,6 +695,8 @@ fn register_sealed(index: &ShardedSegmentIndex, segment: &WalSegment, pipeline_i min_slot: segment.min_slot, max_slot: segment.max_slot, record_count: segment.record_count, + max_term: segment.max_term, + vote_term: segment.vote_term, }; index.register_segment(pipeline_idx, meta); debug!( diff --git a/lib/crowdb-kv/src/wal/record.rs b/lib/crowdb-kv/src/wal/record.rs index 1733296f5..d6e75fa7d 100644 --- a/lib/crowdb-kv/src/wal/record.rs +++ b/lib/crowdb-kv/src/wal/record.rs @@ -253,6 +253,9 @@ impl WALRecord { return Err(RecordError::Truncated); } let frame_len = u32::from_le_bytes([data[0], data[1], data[2], data[3]]) as usize; + if frame_len < HEADER_BODY_LEN + 4 { + return Err(RecordError::Truncated); + } let total = 4 + frame_len; if data.len() < total { return Err(RecordError::Truncated); diff --git a/lib/crowdb-kv/src/wal/replay.rs b/lib/crowdb-kv/src/wal/replay.rs index 53b00bbc1..30f0b52ef 100644 --- a/lib/crowdb-kv/src/wal/replay.rs +++ b/lib/crowdb-kv/src/wal/replay.rs @@ -20,10 +20,12 @@ use super::record::{RecordType, WALRecord}; use super::segment::{parse_segment_filename, SegmentReader}; use super::{IoBackend, OpenOptions}; +mod records; + /// Result of replaying all WAL segments for a group. #[derive(Debug)] pub struct ReplayResult { - /// Verified records in slot order (deduplicated: highest ballot per slot wins). + /// Vote records followed by slot-ordered highest accepted and promised ballots. pub records: Vec, /// Rebuilt segment index. pub index: SegmentIndex, @@ -87,7 +89,7 @@ pub async fn replay_group( ); // ── Step 2–3: scan records, verify CRC, truncate on error ── - let mut verified_records: Vec = Vec::new(); + let mut verified_records = records::ReplayRecords::default(); let mut index = SegmentIndex::new(); let mut max_segment_id: u64 = 0; let total_segments = all_segments.len(); @@ -119,6 +121,8 @@ pub async fn replay_group( let mut seg_min_slot = u64::MAX; let mut seg_max_slot = 0u64; let mut seg_record_count = 0u32; + let mut max_term = 0; + let mut vote_term = None::; loop { match reader.next_record().await { @@ -133,6 +137,10 @@ pub async fn replay_group( } } seg_record_count += 1; + max_term = max_term.max(record.term); + if record.record_type == RecordType::VoteGranted { + vote_term = Some(vote_term.map_or(record.term, |term| term.max(record.term))); + } verified_records.push(record); } Ok(None) => break, // EOF or footer @@ -170,6 +178,8 @@ pub async fn replay_group( min_slot: if seg_min_slot == u64::MAX { 0 } else { seg_min_slot }, max_slot: seg_max_slot, record_count: seg_record_count, + max_term, + vote_term, }; for (slot, file_offset) in &seg_records { index.insert( @@ -186,25 +196,16 @@ pub async fn replay_group( debug!( group_id, - total_records = verified_records.len(), + total_records = verified_records.scanned, max_segment_id, "replay: scan complete" ); - // ── Step 4–7: rebuild in-memory state ────────────── - - // 4. current_term = max term across all records. - let current_term = verified_records.iter().map(|r| r.term).max().unwrap_or(0); - - // 5. voted_for from latest VoteGranted for current_term. - let voted_for = verified_records - .iter() - .rev() - .find(|r| r.record_type == RecordType::VoteGranted && r.term == current_term) - .and_then(WALRecord::voted_for_id); + let current_term = verified_records.current_term; + let voted_for = verified_records.voted_for; Ok(ReplayResult { - records: verified_records, + records: verified_records.into_records(), index, max_segment_id, current_term, diff --git a/lib/crowdb-kv/src/wal/replay/records.rs b/lib/crowdb-kv/src/wal/replay/records.rs new file mode 100644 index 000000000..43e6ef3dc --- /dev/null +++ b/lib/crowdb-kv/src/wal/replay/records.rs @@ -0,0 +1,53 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::collections::BTreeMap; + +use crate::paxos::{PxNodeId, PxTerm}; +use crate::wal::record::{RecordType, WALRecord}; + +#[derive(Default)] +pub(super) struct ReplayRecords { + slots: BTreeMap<(u64, u8), WALRecord>, + votes: Vec, + pub(super) current_term: PxTerm, + pub(super) voted_for: Option, + pub(super) scanned: u64, +} + +impl ReplayRecords { + pub(super) fn push(&mut self, record: WALRecord) { + self.scanned += 1; + // Election safety depends on every verified record, including accepts + // superseded by a higher ballot and records physically ordered by disk. + if record.term > self.current_term { + self.current_term = record.term; + self.voted_for = None; + } + if record.record_type == RecordType::VoteGranted { + if record.term == self.current_term { + self.voted_for = record.voted_for_id(); + } + self.votes.push(record); + return; + } + // A later promise must never erase an older accepted value. Retain the + // highest accepted ballot and the highest promised ballot independently. + let key = (record.slot, record.record_type as u8); + match self.slots.entry(key) { + std::collections::btree_map::Entry::Vacant(entry) => { + entry.insert(record); + } + std::collections::btree_map::Entry::Occupied(mut entry) => { + if record.ballot >= entry.get().ballot { + entry.insert(record); + } + } + } + } + + pub(super) fn into_records(mut self) -> Vec { + self.votes.extend(self.slots.into_values()); + self.votes + } +} diff --git a/lib/crowdb-kv/src/wal/segment.rs b/lib/crowdb-kv/src/wal/segment.rs index 55bc4bc1d..18675afcb 100644 --- a/lib/crowdb-kv/src/wal/segment.rs +++ b/lib/crowdb-kv/src/wal/segment.rs @@ -39,6 +39,8 @@ use crate::paxos::PxGroupId; use super::record::{parse_hex_u32, RecordError, WALRecord, WalRecordFormat}; use super::{IoBackend, OpenOptions, WalFile}; +mod read_buffer; + const SEG_MAGIC: u32 = 0x5345_474D; const SEG_VERSION: u16 = 1; pub const SEG_HEADER_LEN: usize = 4 + 2 + 8 + 8; // 22 @@ -68,6 +70,8 @@ pub struct WalSegment { pub min_slot: SlotIndex, pub max_slot: SlotIndex, pub record_count: u32, + pub(super) max_term: u64, + pub(super) vote_term: Option, sealed: bool, record_format: WalRecordFormat, } @@ -126,6 +130,8 @@ impl WalSegment { min_slot: u64::MAX, max_slot: 0, record_count: 0, + max_term: 0, + vote_term: None, sealed: false, record_format, }) @@ -159,6 +165,10 @@ impl WalSegment { } } self.record_count += 1; + self.max_term = self.max_term.max(record.term); + if record.record_type == super::record::RecordType::VoteGranted { + self.vote_term = Some(self.vote_term.map_or(record.term, |term| term.max(record.term))); + } Ok(offset) } @@ -299,6 +309,7 @@ pub struct SegmentReader { offset: u64, /// File size (to detect truncation). file_len: u64, + buffer: read_buffer::SegmentReadBuffer, } impl SegmentReader { @@ -347,6 +358,7 @@ impl SegmentReader { header, offset: data_offset, file_len, + buffer: read_buffer::SegmentReadBuffer::default(), }) } else { // Binary-format header. @@ -359,6 +371,7 @@ impl SegmentReader { header, offset: SEG_HEADER_LEN as u64, file_len, + buffer: read_buffer::SegmentReadBuffer::default(), }) } } @@ -409,12 +422,12 @@ impl SegmentReader { // Peek the next frame marker (the `frame_len` of a record, the // `FOOTER_MAGIC` of a seal footer, or `0` for trailing padding). - let mut frame_hdr = [0u8; 4]; - self.file - .read_exact_at(&mut frame_hdr, self.offset) + let frame_hdr = self + .buffer + .read(&mut self.file, self.file_len, self.offset, 4) .await .map_err(|_| (RecordError::Truncated, self.offset))?; - let marker = u32::from_le_bytes(frame_hdr); + let marker = u32::from_le_bytes(frame_hdr.as_ref().try_into().expect("four-byte frame prefix")); // Footer reached → records are done (sealed segment). if marker == FOOTER_MAGIC { @@ -436,9 +449,14 @@ impl SegmentReader { return Err((RecordError::Truncated, self.offset)); } - let mut buf = vec![0u8; usize::try_from(total).expect("record size exceeds usize")]; - self.file - .read_exact_at(&mut buf, self.offset) + let buf = self + .buffer + .read( + &mut self.file, + self.file_len, + self.offset, + usize::try_from(total).expect("record size exceeds usize"), + ) .await .map_err(|_| (RecordError::Truncated, self.offset))?; @@ -456,12 +474,12 @@ impl SegmentReader { if self.file_len - self.offset < prefix.len() as u64 { return Ok(false); } - let mut buf = vec![0u8; prefix.len()]; - self.file - .read_exact_at(&mut buf, self.offset) + let buf = self + .buffer + .read(&mut self.file, self.file_len, self.offset, prefix.len()) .await .map_err(|_| (RecordError::Truncated, self.offset))?; - Ok(buf == prefix) + Ok(buf.as_ref() == prefix) } async fn next_is_text_footer(&mut self) -> Result { @@ -469,12 +487,12 @@ impl SegmentReader { if self.file_len - self.offset < prefix.len() as u64 { return Ok(false); } - let mut buf = vec![0u8; prefix.len()]; - self.file - .read_exact_at(&mut buf, self.offset) + let buf = self + .buffer + .read(&mut self.file, self.file_len, self.offset, prefix.len()) .await .map_err(|_| (RecordError::Truncated, self.offset))?; - Ok(buf == prefix) + Ok(buf.as_ref() == prefix) } async fn next_text_record( diff --git a/lib/crowdb-kv/src/wal/segment/read_buffer.rs b/lib/crowdb-kv/src/wal/segment/read_buffer.rs new file mode 100644 index 000000000..c8fe2d040 --- /dev/null +++ b/lib/crowdb-kv/src/wal/segment/read_buffer.rs @@ -0,0 +1,50 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::io; + +use bytes::Bytes; + +use crate::wal::WalFile; + +const READ_AHEAD: usize = 1024 * 1024; + +#[derive(Default)] +pub(super) struct SegmentReadBuffer { + bytes: Bytes, + offset: u64, +} + +impl SegmentReadBuffer { + pub(super) async fn read( + &mut self, + file: &mut WalFile, + file_len: u64, + offset: u64, + length: usize, + ) -> io::Result { + if let Some(start) = offset + .checked_sub(self.offset) + .and_then(|start| usize::try_from(start).ok()) + { + if start <= self.bytes.len() && length <= self.bytes.len() - start { + return Ok(self.bytes.slice(start..start + length)); + } + } + let available = file_len + .checked_sub(offset) + .ok_or_else(|| io::Error::new(io::ErrorKind::UnexpectedEof, "WAL read starts past EOF"))?; + if u64::try_from(length).unwrap_or(u64::MAX) > available { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "WAL read extends past EOF", + )); + } + let read_ahead = usize::try_from(available.min(READ_AHEAD as u64)).expect("bounded read size"); + let mut bytes = vec![0; length.max(read_ahead)]; + file.read_exact_at(&mut bytes, offset).await?; + self.bytes = Bytes::from(bytes); + self.offset = offset; + Ok(self.bytes.slice(..length)) + } +} diff --git a/lib/crowdb-kv/src/wal/wal_engine.rs b/lib/crowdb-kv/src/wal/wal_engine.rs index b04151a43..c44a4b4b5 100644 --- a/lib/crowdb-kv/src/wal/wal_engine.rs +++ b/lib/crowdb-kv/src/wal/wal_engine.rs @@ -21,7 +21,7 @@ use crate::metrics::{Bandwidth, Counter, LatencySummary}; use crate::paxos::roles::SlotIndex; use crate::paxos::PxGroupId; -use super::index::{ShardedSegmentIndex, SlotLocation}; +use super::index::{SegmentIndex, ShardedSegmentIndex, SlotLocation}; use super::pipeline::WalPipeline; use super::pipeline_backend::{WalBlockAlignment, WalPipelineBackend}; use super::pipeline_writer::{spawn_pipeline_writer, EncodedRecord, PendingWrite, WriterCommand}; @@ -153,10 +153,48 @@ impl WalEngine { config: WalConfig, group_id: PxGroupId, initial_next_segment_id: u64, + ) -> io::Result> { + Self::create_with_index( + backend, + config, + group_id, + initial_next_segment_id, + SegmentIndex::new(), + ) + .await + } + + /// Resume a WAL with the verified index and next segment id recovered locally. + /// + /// # Errors + /// Returns an error if recovered disks are unavailable or writers cannot start. + pub async fn create_from_replay( + backend: Arc, + config: WalConfig, + group_id: PxGroupId, + max_segment_id: u64, + index: SegmentIndex, + ) -> io::Result> { + let next_segment_id = max_segment_id + .checked_add(1) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "WAL segment id exhausted"))?; + Self::create_with_index(backend, config, group_id, next_segment_id.max(1), index).await + } + + async fn create_with_index( + backend: Arc, + config: WalConfig, + group_id: PxGroupId, + initial_next_segment_id: u64, + recovered_index: SegmentIndex, ) -> io::Result> { let pipeline_count = config.wal_disks.len(); let failed = Arc::new(AtomicBool::new(false)); - let index = Arc::new(ShardedSegmentIndex::new(pipeline_count, group_id)); + let index = Arc::new(ShardedSegmentIndex::from_replay( + pipeline_count, + group_id, + recovered_index, + )?); let next_segment_id = Arc::new(AtomicU64::new(initial_next_segment_id)); let flush_count = Arc::new(AtomicU64::new(0)); let records_flushed = Arc::new(AtomicU64::new(0)); @@ -282,6 +320,8 @@ impl WalEngine { let pending = PendingWrite { encoded, slot: record.slot, + term: record.term, + is_vote: record.record_type == super::record::RecordType::VoteGranted, ack: ack_tx, }; diff --git a/lib/crowdb-kv/src/wal/wal_file.rs b/lib/crowdb-kv/src/wal/wal_file.rs index 80bf288b4..36c704009 100644 --- a/lib/crowdb-kv/src/wal/wal_file.rs +++ b/lib/crowdb-kv/src/wal/wal_file.rs @@ -39,7 +39,7 @@ impl WalFile { WalFileInner::File(f) => f.write_at(data, offset).await, WalFileInner::Uring(f) => f.write_at(data, offset).await, WalFileInner::MemBlock(f) => f.write_at(data, offset), - WalFileInner::Block(f) => f.write_at(data, offset), + WalFileInner::Block(f) => block_backend::blocking_io(|| f.write_at(data, offset)), } } @@ -58,7 +58,7 @@ impl WalFile { WalFileInner::File(f) => f.write_vectored_at(bufs, offset).await, WalFileInner::Uring(f) => f.write_vectored_at(bufs, offset).await, WalFileInner::MemBlock(f) => f.write_vectored_at(bufs, offset), - WalFileInner::Block(f) => f.write_vectored_at(bufs, offset), + WalFileInner::Block(f) => block_backend::blocking_io(|| f.write_vectored_at(bufs, offset)), } } @@ -71,7 +71,7 @@ impl WalFile { WalFileInner::File(f) => f.read_at(buf, offset).await, WalFileInner::Uring(f) => f.read_at(buf, offset).await, WalFileInner::MemBlock(f) => f.read_at(buf, offset), - WalFileInner::Block(f) => f.read_at(buf, offset), + WalFileInner::Block(f) => block_backend::blocking_io(|| f.read_at(buf, offset)), } } @@ -84,7 +84,7 @@ impl WalFile { WalFileInner::File(f) => f.read_exact_at(buf, offset).await, WalFileInner::Uring(f) => f.read_exact_at(buf, offset).await, WalFileInner::MemBlock(f) => f.read_exact_at(buf, offset), - WalFileInner::Block(f) => f.read_exact_at(buf, offset), + WalFileInner::Block(f) => block_backend::blocking_io(|| f.read_exact_at(buf, offset)), } } @@ -97,7 +97,7 @@ impl WalFile { WalFileInner::File(f) => f.fdatasync().await, WalFileInner::Uring(f) => f.fdatasync().await, WalFileInner::MemBlock(f) => f.fdatasync(), - WalFileInner::Block(f) => f.fdatasync(), + WalFileInner::Block(f) => block_backend::blocking_io(|| f.fdatasync()), } } @@ -110,7 +110,7 @@ impl WalFile { WalFileInner::File(f) => f.fsync().await, WalFileInner::Uring(f) => f.fsync().await, WalFileInner::MemBlock(f) => f.fsync(), - WalFileInner::Block(f) => f.fsync(), + WalFileInner::Block(f) => block_backend::blocking_io(|| f.fsync()), } } @@ -133,7 +133,7 @@ impl WalFile { WalFileInner::File(f) => f.len().await, WalFileInner::Uring(f) => f.len(), WalFileInner::MemBlock(f) => f.len(), - WalFileInner::Block(f) => f.len(), + WalFileInner::Block(f) => block_backend::blocking_io(|| f.len()), } } @@ -146,7 +146,7 @@ impl WalFile { WalFileInner::File(f) => f.truncate(len).await, WalFileInner::Uring(f) => f.truncate(len), WalFileInner::MemBlock(f) => f.truncate(len), - WalFileInner::Block(f) => f.truncate(len), + WalFileInner::Block(f) => block_backend::blocking_io(|| f.truncate(len)), } } } diff --git a/lib/crowdb-kv/tests/election_test/vote_test.rs b/lib/crowdb-kv/tests/election_test/vote_test.rs index dbbb1e251..6abb023d2 100644 --- a/lib/crowdb-kv/tests/election_test/vote_test.rs +++ b/lib/crowdb-kv/tests/election_test/vote_test.rs @@ -9,7 +9,85 @@ use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; use crowdb_kv::cluster::replica::{ReplicaHandler, VoteRequestPayload}; +use crowdb_kv::common::config::PxElectionConfig; use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; +use std::time::Instant; + +#[tokio::test] +async fn quorum_leased_leader_rejects_prevote_after_its_self_vote_lockout_expires() { + let replica = PxLocalReplica::new(1, PxLocalReplicaRole::Follower); + replica.become_candidate(1); + replica.become_leader(); + // The original self-vote lockout expires during a long healthy tenure; + // quorum heartbeats continue renewing only the lock-free leader lease. + replica.clear_vote_lockout(); + replica.renew_lease(Instant::now(), &PxElectionConfig::DEFAULT); + let request = make_vote_req(2, 2, 0, 0); + let reply = replica.on_pre_vote(request, 1).await.unwrap(); + assert!( + !reply.granted, + "a valid quorum lease must protect the leader's own vote" + ); + assert_eq!(replica.current_term_snapshot(), 1); + assert_eq!(replica.role(), PxLocalReplicaRole::Leader); + + replica.reset_lease_to(Instant::now()); + assert!(replica.on_pre_vote(request, 1).await.unwrap().granted); +} + +#[tokio::test] +async fn quorum_leased_leader_rejects_request_vote_after_its_self_vote_lockout_expires() { + let replica = PxLocalReplica::new(1, PxLocalReplicaRole::Follower); + replica.become_candidate(1); + replica.become_leader(); + replica.clear_vote_lockout(); + replica.renew_lease(Instant::now(), &PxElectionConfig::DEFAULT); + let request = make_vote_req(2, 2, 0, 0); + let reply = replica.on_request_vote(request, 1).await.unwrap(); + assert!( + !reply.granted, + "a challenger must not depose a quorum-leased leader" + ); + assert_eq!(replica.current_term_snapshot(), 1); + assert_eq!(replica.role(), PxLocalReplicaRole::Leader); + + replica.reset_lease_to(Instant::now()); + assert!(replica.on_request_vote(request, 1).await.unwrap().granted); + assert_eq!(replica.current_term_snapshot(), 2); + assert_eq!(replica.role(), PxLocalReplicaRole::Follower); +} + +#[tokio::test] +async fn two_lagging_voters_cannot_use_the_healthy_leader_to_form_a_prevote_quorum() { + let voters: Vec<_> = (1..=5) + .map(|id| PxLocalReplica::new(id, PxLocalReplicaRole::Follower)) + .collect(); + let leader = &voters[0]; + leader.become_candidate(1); + leader.become_leader(); + leader.clear_vote_lockout(); + leader.renew_lease(Instant::now(), &PxElectionConfig::DEFAULT); + for follower in &voters[1..3] { + assert!( + follower + .on_request_vote(make_vote_req(1, 1, 0, 0), 1) + .await + .unwrap() + .granted + ); + } + for lagging in &voters[3..] { + lagging.become_follower(1); + lagging.clear_vote_lockout(); + } + let request = make_vote_req(2, 5, 0, 0); + let mut grants = 1; // Challenger's own pre-vote. + for peer in &voters[..4] { + grants += usize::from(peer.on_pre_vote(request, 1).await.unwrap().granted); + } + assert_eq!(grants, 2, "only the isolated minority may grant this pre-vote"); + assert_eq!(leader.role(), PxLocalReplicaRole::Leader); +} fn make_vote_req(term: u64, candidate_id: u64, tip_slot: u64, tip_term: u64) -> VoteRequestPayload { VoteRequestPayload { diff --git a/lib/crowdb-kv/tests/group_test.rs b/lib/crowdb-kv/tests/group_test.rs index 274c7090b..f4a7aa059 100644 --- a/lib/crowdb-kv/tests/group_test.rs +++ b/lib/crowdb-kv/tests/group_test.rs @@ -86,6 +86,12 @@ mod g4_learner_stream; #[path = "group_test/g5_recovery_test.rs"] mod g5_recovery; +#[path = "group_test/recovery_backlog_test.rs"] +mod recovery_backlog; + +#[path = "group_test/fetch_gap_durability_test.rs"] +mod fetch_gap_durability; + #[path = "group_test/g6_reconfig_test.rs"] mod g6_reconfig; diff --git a/lib/crowdb-kv/tests/group_test/fetch_gap_durability_test.rs b/lib/crowdb-kv/tests/group_test/fetch_gap_durability_test.rs new file mode 100644 index 000000000..50422cdcb --- /dev/null +++ b/lib/crowdb-kv/tests/group_test/fetch_gap_durability_test.rs @@ -0,0 +1,153 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crate::common::cluster::start_cluster; +use crowdb_kv::{ + cluster::{group::PxGroup, KvServer, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}, + paxos::roles::{Acceptor, Learner, PxBallot, PxLogEntry}, + wal::{replay::replay_group, IoBackend, MemBlockDevice, WalConfig, WalEngine}, +}; +use std::{path::PathBuf, sync::Arc, time::Duration}; + +#[tokio::test] +async fn fetched_historical_chosen_value_survives_a_follower_restart() { + restart_fetched_value(false).await; +} + +#[tokio::test] +async fn learned_historical_value_preserves_a_higher_promise_across_restart() { + restart_fetched_value(true).await; +} + +async fn restart_fetched_value(higher_promise: bool) { + let cluster = start_cluster(&[1, 2], 1).await; + let leader = cluster.leader().get_group(1).unwrap(); + let entry = PxLogEntry { + slot: 1, + ballot: PxBallot::new(1, 1), + term: 0, + payload: bytes::Bytes::from_static(b"\x01\x00\x00\x03\x00\x00\x00key\x05\x00\x00\x00value"), + }; + leader.local_replica().acceptor.accept(&entry).await; + leader.local_replica().learner.learn(entry.clone(), &[]).await; + let backend = Arc::new(IoBackend::MemBlock(MemBlockDevice::new())); + let disks = vec![PathBuf::from("/fetch-gap")]; + let wal = WalEngine::create( + backend.clone(), + WalConfig { + wal_disks: disks.clone(), + ..Default::default() + }, + 1, + ) + .await + .unwrap(); + let mut replica = PxLocalReplica::new(99, PxLocalReplicaRole::Follower); + replica.become_follower(7); + replica.set_believed_leader(1); + if higher_promise { + let stale = PxLogEntry { + ballot: PxBallot::new(0, 99), + payload: bytes::Bytes::from_static(b"\x01\x00\x00\x03\x00\x00\x00key\x05\x00\x00\x00stale"), + ..entry.clone() + }; + replica.acceptor.accept(&stale).await; + wal.append(&crowdb_kv::wal::WALRecord::from_accepted(1, &stale)) + .await + .unwrap(); + let promise = PxBallot::new(9, 99); + replica.acceptor.prepare(1, promise).await; + wal.append(&crowdb_kv::wal::WALRecord::from_promised(1, 7, 1, promise)) + .await + .unwrap(); + } + replica.set_wal(wal.clone()); + replica.persist_current_vote().await; + let mut group = PxGroup::new(1, replica); + group.set_remote_replicas(vec![PxRemoteReplica::new( + 1, + cluster.leader().listen_addr().unwrap().to_string(), + )]); + group + .get_remote_replica(1) + .unwrap() + .set_rpc_transport(cluster.leader().rpc_transport().unwrap()); + let group = Arc::new(group); + group.start_fetchgap_driver().await; + group.note_missing_chosen_for_tests(1); + tokio::time::timeout(Duration::from_secs(3), async { + while group.local_replica().contiguous_applied() != 1 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!( + group.local_replica().learner.engine_get(b"key").await, + Some((1, b"value".to_vec())) + ); + assert_eq!( + group.local_replica().promised_at(1).await, + Some(if higher_promise { + PxBallot::new(9, 99) + } else { + entry.ballot + }) + ); + group.shutdown(Duration::from_secs(3)).await; + wal.seal_all().await.unwrap(); + let replay = replay_group(&backend, &disks, 1).await.unwrap(); + assert!( + replay + .records + .iter() + .any(|record| record.to_log_entry() == Some(entry.clone())), + "FetchGap advanced the applied frontier without persisting the recovered entry" + ); + let restored = PxLocalReplica::restore_from_replay(99, PxLocalReplicaRole::Follower, &replay) + .await + .unwrap(); + assert_eq!(restored.contiguous_applied(), 1); + assert_eq!(restored.current_term_snapshot(), 7); + assert_eq!( + restored.learner.engine_get(b"key").await, + Some((1, b"value".to_vec())) + ); + assert_eq!(restored.accepted_at(1).await, Some(entry)); + if higher_promise { + assert_eq!(restored.promised_at(1).await, Some(PxBallot::new(9, 99))); + } + cluster.shutdown().await; +} + +#[tokio::test] +async fn fetch_gap_does_not_advertise_an_unproven_accepted_value_as_chosen() { + let cluster = start_cluster(&[1, 2], 1).await; + let leader = cluster.leader().get_group(1).unwrap(); + let entry = PxLogEntry { + slot: 1, + ballot: PxBallot::new(1, 1), + term: 0, + payload: bytes::Bytes::new(), + }; + leader.local_replica().acceptor.accept(&entry).await; + let transport = cluster.leader().rpc_transport().unwrap(); + let endpoint = cluster.leader().listen_addr().unwrap().to_string(); + let destination = crowdb_kv::rpc::ReplicaDestination { + endpoint: &endpoint, + replica_id: 1, + }; + assert!( + transport.send_fetch_gap(destination, 1, 1, 0, 1).await.is_err(), + "an accepted value without a chosen proof must not be advertised as chosen" + ); + leader.local_replica().learner.learn(entry.clone(), &[]).await; + let destination = crowdb_kv::rpc::ReplicaDestination { + endpoint: &endpoint, + replica_id: 1, + }; + let reply = transport.send_fetch_gap(destination, 1, 1, 0, 1).await.unwrap(); + assert_eq!(reply.slot, entry.slot); + assert_eq!(reply.payload, entry.payload); + cluster.shutdown().await; +} diff --git a/lib/crowdb-kv/tests/group_test/g3_leader_change_test.rs b/lib/crowdb-kv/tests/group_test/g3_leader_change_test.rs index 5401eaf69..1965f3d88 100644 --- a/lib/crowdb-kv/tests/group_test/g3_leader_change_test.rs +++ b/lib/crowdb-kv/tests/group_test/g3_leader_change_test.rs @@ -12,6 +12,7 @@ use std::time::{Duration, Instant}; use bytes::Bytes; +use crowdb_kv::cluster::group_election::LeaderElection; use crowdb_kv::cluster::replica::StepDownRequestPayload; use crowdb_kv::rpc::{KvGetRequest, KvSetRequest}; @@ -21,7 +22,13 @@ async fn wait_for_leader(cluster: &TestCluster, timeout: Duration) -> Option previous_term, + "the real driver must complete a new election" + ); + assert_eq!( + role, + PxLocalReplicaRole::Leader, + "a prior-tenure admin wakeup stepped down the new leader" + ); + assert!(fresh_step_down.unwrap().accepted); + assert_eq!(replica.role(), PxLocalReplicaRole::Follower); +} diff --git a/lib/crowdb-kv/tests/group_test/recovery_backlog_test.rs b/lib/crowdb-kv/tests/group_test/recovery_backlog_test.rs new file mode 100644 index 000000000..14f5dbf41 --- /dev/null +++ b/lib/crowdb-kv/tests/group_test/recovery_backlog_test.rs @@ -0,0 +1,92 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crate::common::cluster::start_cluster; +use crowdb_kv::{ + cluster::group_election::LeaderElection, + common::config::PxElectionConfig, + paxos::roles::{Acceptor, Learner, PxBallot, PxLogEntry}, +}; +use std::time::Duration; +use tokio_util::sync::CancellationToken; + +#[tokio::test] +async fn bulk_recovery_fills_its_bounded_window_before_waiting_for_peer_io() { + let cluster = start_cluster(&[1, 2, 3], 1).await; + let group = cluster.leader().get_group(1).unwrap(); + let config = PxElectionConfig { + bulk_prepare_window: 4, + ..group.election_config() + }; + let recovery = group.run_bulk_phase1(0, 0, 12, config, CancellationToken::new()); + tokio::pin!(recovery); + // On this current-thread runtime the peers cannot answer until this poll + // yields. Slow IO at the first slot must leave all four slots dispatched. + for _ in 0..16 { + assert!(futures::poll!(&mut recovery).is_pending()); + if group.local_replica().highest_seen_slot() == 4 { + break; + } + } + assert_eq!(group.local_replica().highest_seen_slot(), 4); + tokio::time::timeout(Duration::from_secs(3), recovery) + .await + .expect("bounded recovery must finish"); + assert_eq!(group.local_replica().contiguous_applied(), 12); + assert!(group.leader_read_ready()); + cluster.shutdown().await; +} + +#[tokio::test] +async fn cancelling_a_pending_bulk_window_does_not_accept_its_prepared_slots() { + let cluster = start_cluster(&[1, 2, 3], 1).await; + let group = cluster.leader().get_group(1).unwrap(); + let config = PxElectionConfig { + bulk_prepare_window: 4, + ..group.election_config() + }; + let cancel = CancellationToken::new(); + let recovery = group.run_bulk_phase1(0, 0, 12, config, cancel.clone()); + tokio::pin!(recovery); + assert!(futures::poll!(&mut recovery).is_pending()); + cancel.cancel(); + recovery.await; + assert_eq!(group.local_replica().contiguous_applied(), 0); + for slot in 1..=12 { + assert!(group.local_replica().accepted_at(slot).await.is_none()); + } + cluster.shutdown().await; +} + +#[tokio::test] +async fn follower_fetches_a_backlog_above_the_unimplemented_snapshot_threshold() { + let cluster = start_cluster(&[1, 2, 3], 1).await; + let leader = cluster.leader().get_group(1).unwrap(); + let follower = cluster.followers()[0].get_group(1).unwrap(); + let last = leader.election_config().catchup_snapshot_threshold + 1; + for slot in 1..=last { + let entry = PxLogEntry { + slot, + ballot: PxBallot::new(1, 1), + term: 0, + payload: bytes::Bytes::new(), + }; + leader.local_replica().acceptor.accept(&entry).await; + leader.local_replica().learner.learn(entry, &[]).await; + follower.note_missing_chosen_for_tests(slot); + } + tokio::time::timeout(Duration::from_secs(3), async { + while follower.local_replica().contiguous_applied() != last { + tokio::task::yield_now().await; + } + }) + .await + .expect("snapshot threshold must not permanently disable bounded catch-up"); + for slot in 1..=last { + assert_eq!( + follower.local_replica().accepted_at(slot).await, + leader.local_replica().accepted_at(slot).await + ); + } + cluster.shutdown().await; +} diff --git a/lib/crowdb-kv/tests/replay_compaction_memory_test.rs b/lib/crowdb-kv/tests/replay_compaction_memory_test.rs new file mode 100644 index 000000000..55caf9424 --- /dev/null +++ b/lib/crowdb-kv/tests/replay_compaction_memory_test.rs @@ -0,0 +1,82 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#![cfg(target_os = "linux")] + +use std::io::Write; +use std::sync::Arc; + +use bytes::Bytes; +use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_kv::wal::record::WALRecord; +use crowdb_kv::wal::replay::replay_group; +use crowdb_kv::wal::IoBackend; + +fn peak_rss_kib() -> u64 { + std::fs::read_to_string("/proc/self/status") + .unwrap() + .lines() + .find_map(|line| line.strip_prefix("VmHWM:")) + .unwrap() + .split_whitespace() + .next() + .unwrap() + .parse() + .unwrap() +} + +#[tokio::test] +async fn replay_releases_superseded_accepts_while_scanning_half_a_gibibyte() { + let temp = crowdb_test_harness::test_dirs::tempdir_in_test_data("wal-compaction-memory"); + let directory = temp.path().join("group1"); + std::fs::create_dir(&directory).unwrap(); + let mut file = std::io::BufWriter::new(std::fs::File::create(directory.join("seg-0000001.ck")).unwrap()); + for bytes in [ + &0x5345_474D_u32.to_le_bytes()[..], + &1_u16.to_le_bytes(), + &1_u64.to_le_bytes(), + &1_u64.to_le_bytes(), + ] { + file.write_all(bytes).unwrap(); + } + for sequence in 1_u64..=8192 { + let round = (sequence - 1) / 32 + 1; + let entry = PxLogEntry { + slot: (sequence - 1) % 32 + 1, + ballot: PxBallot::new(round, 7), + term: round, + payload: Bytes::from(vec![u8::try_from(sequence % 251).unwrap(); 65536]), + }; + file.write_all(&WALRecord::from_accepted(1, &entry).encode()) + .unwrap(); + } + file.flush().unwrap(); + drop(file); + let before = peak_rss_kib(); + let result = replay_group( + &Arc::new(IoBackend::block_device()), + &[temp.path().to_path_buf()], + 1, + ) + .await + .unwrap(); + let growth = peak_rss_kib().saturating_sub(before); + println!("Compacted replay peak growth: {growth} KiB for 512 MiB of accepted payloads"); + assert!( + growth < 64 * 1024, + "superseded accepts remained resident during replay: {growth} KiB" + ); + assert_eq!(result.records.len(), 32); + assert_eq!(result.current_term, 256); + assert_eq!(result.index.slot_count(), 32); + assert_eq!(result.index.segments().next().unwrap().record_count, 8192); + for (index, record) in result.records.iter().enumerate() { + let slot = u64::try_from(index).unwrap() + 1; + assert_eq!(record.slot, slot); + assert_eq!(record.ballot, PxBallot::new(256, 7)); + assert_eq!( + record.payload, + vec![u8::try_from((8160 + slot) % 251).unwrap(); 65536] + ); + } +} diff --git a/lib/crowdb-kv/tests/replay_memory_test.rs b/lib/crowdb-kv/tests/replay_memory_test.rs new file mode 100644 index 000000000..010ccadde --- /dev/null +++ b/lib/crowdb-kv/tests/replay_memory_test.rs @@ -0,0 +1,85 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#![cfg(target_os = "linux")] + +use std::io::Write; +use std::sync::Arc; + +use bytes::Bytes; +use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_kv::wal::record::WALRecord; +use crowdb_kv::wal::replay::replay_group; +use crowdb_kv::wal::IoBackend; + +fn peak_rss_kib() -> u64 { + std::fs::read_to_string("/proc/self/status") + .unwrap() + .lines() + .find_map(|line| line.strip_prefix("VmHWM:")) + .unwrap() + .split_whitespace() + .next() + .unwrap() + .parse() + .unwrap() +} + +#[tokio::test] +async fn binary_replay_keeps_small_record_history_within_one_gibibyte() { + const COUNT: u64 = 800_000; + const PAYLOAD: usize = 768; + let temp = crowdb_test_harness::test_dirs::tempdir_in_test_data("wal-small-record-memory"); + let directory = temp.path().join("group1"); + std::fs::create_dir(&directory).unwrap(); + let mut file = std::io::BufWriter::new(std::fs::File::create(directory.join("seg-0000001.ck")).unwrap()); + file.write_all(&0x5345_474D_u32.to_le_bytes()).unwrap(); + file.write_all(&1_u16.to_le_bytes()).unwrap(); + file.write_all(&1_u64.to_le_bytes()).unwrap(); + file.write_all(&1_u64.to_le_bytes()).unwrap(); + for slot in 1..=COUNT { + let entry = PxLogEntry { + slot, + ballot: PxBallot::new(7, 1), + term: 7, + payload: Bytes::from(vec![u8::try_from(slot % 251).unwrap(); PAYLOAD]), + }; + file.write_all(&WALRecord::from_accepted(1, &entry).encode()) + .unwrap(); + } + file.flush().unwrap(); + drop(file); + let before = peak_rss_kib(); + let started = std::time::Instant::now(); + let result = replay_group( + &Arc::new(IoBackend::block_device()), + &[temp.path().to_path_buf()], + 1, + ) + .await + .unwrap(); + let growth = peak_rss_kib().saturating_sub(before); + println!( + "Replay {} records / {} payload bytes: peak growth {} KiB; elapsed {:?}", + COUNT, + COUNT * PAYLOAD as u64, + growth, + started.elapsed() + ); + assert_eq!(result.records.len(), usize::try_from(COUNT).unwrap()); + assert_eq!(result.index.slot_count(), usize::try_from(COUNT).unwrap()); + assert_eq!(result.current_term, 7); + for (index, record) in result.records.iter().enumerate() { + let slot = u64::try_from(index).unwrap() + 1; + assert_eq!(record.slot, slot); + assert_eq!(record.payload.len(), PAYLOAD); + assert!(record + .payload + .iter() + .all(|byte| *byte == u8::try_from(slot % 251).unwrap())); + } + assert!( + growth < 1024 * 1024, + "small-record WAL replay exceeded its 1 GiB memory budget: {growth} KiB" + ); +} diff --git a/lib/crowdb-kv/tests/rpc_migration_test.rs b/lib/crowdb-kv/tests/rpc_migration_test.rs index d54782206..171d2fb33 100644 --- a/lib/crowdb-kv/tests/rpc_migration_test.rs +++ b/lib/crowdb-kv/tests/rpc_migration_test.rs @@ -303,10 +303,17 @@ async fn crowdb_rpc_fetch_gap() { .send_prepare(1, ballot, 0, 1, 0) .await .expect("follower prepare"); - follower_remote + let accepted = follower_remote .send_accept(&entry, 1, 0) .await .expect("follower accept"); + assert!(matches!( + accepted, + crowdb_kv::paxos::roles::PxAcceptReply::Accepted { .. } + )); + // Both voters have accepted. Publish that quorum decision before serving + // the chosen value; an Accepted response alone is not a learner proof. + leader_replica.learn_chosen(&entry, &[]).await; // Use the follower's remote (points at leader) to fetch gap. let follower_group = cluster.follower.get_group(1).expect("follower group"); diff --git a/lib/crowdb-kv/tests/store_test.rs b/lib/crowdb-kv/tests/store_test.rs index b6ef1439e..784391953 100644 --- a/lib/crowdb-kv/tests/store_test.rs +++ b/lib/crowdb-kv/tests/store_test.rs @@ -61,3 +61,6 @@ mod bounded_scan; #[path = "store_test/group_operations_test.rs"] mod group_operations; + +#[path = "store_test/journal_recovery_test.rs"] +mod journal_recovery; diff --git a/lib/crowdb-kv/tests/store_test/journal_recovery_test.rs b/lib/crowdb-kv/tests/store_test/journal_recovery_test.rs new file mode 100644 index 000000000..61cf2b043 --- /dev/null +++ b/lib/crowdb-kv/tests/store_test/journal_recovery_test.rs @@ -0,0 +1,82 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use bytes::Bytes; +use crowdb_kv::cluster::group::PxGroup; +use crowdb_kv::cluster::kv_store::KvStore; +use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_kv::kv::{Batch, BatchOp, CrowdbTreeConfig, CrowdbTreeEngine, KVEngine, Op}; +use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_kv::rpc::{KvErrorCode, ReadMode}; +use crowdb_kv::wal::index::SegmentIndex; +use crowdb_kv::wal::record::WALRecord; +use crowdb_kv::wal::replay::ReplayResult; + +#[tokio::test] +async fn restored_journal_rejects_missing_durable_history_before_maintenance() { + let engine = CrowdbTreeEngine::open(&CrowdbTreeConfig::default()).unwrap(); + for slot in 1..=3 { + engine + .apply( + slot, + &Batch { + ops: vec![BatchOp { + key: Bytes::from(format!("busy:{slot}")), + op: Op::Put(Bytes::from_static(b"occupied")), + }], + }, + ) + .await + .unwrap(); + } + engine.handle().flush().unwrap(); + assert_eq!(engine.resume_from_slot(), 3); + // Earlier WAL segments were reclaimed after the tree became durable. + // The surviving tail still seeds the applied frontier on restart. + let entry = PxLogEntry { + slot: 3, + ballot: PxBallot::new(0, 1), + term: 1, + payload: Bytes::new(), + }; + let replay = ReplayResult { + records: vec![WALRecord::from_accepted(1, &entry)], + index: SegmentIndex::new(), + max_segment_id: 1, + current_term: 1, + voted_for: None, + }; + let replica = PxLocalReplica::restore_from_replay_with_engine( + 1, + PxLocalReplicaRole::Leader, + &replay, + Box::new(engine), + ) + .await + .unwrap(); + assert_eq!(replica.contiguous_applied(), 3); + assert_eq!(replica.acceptor.trim_slot(), 0); + let store = PxKvStore::new(0, "127.0.0.1:0".parse().unwrap()); + store.add_group(PxGroup::new(1, replica)); + let response = store + .kv_journal_scan(1, 1, 0, b"busy:", 0, ReadMode::Linearizable as i32, 1, 1) + .await; + assert_eq!( + response.error_code, + KvErrorCode::KvErrorJournalScanGcGap as i32, + "missing allocation history must select full-scan recovery: {response:?}" + ); + assert!(!response.ok); + let retained_tail = store + .kv_journal_scan(1, 3, 0, b"busy:", 0, ReadMode::Linearizable as i32, 3, 3) + .await; + assert!( + retained_tail.ok, + "the retained NoOp is complete history: {retained_tail:?}" + ); + assert!(retained_tail.ops.is_empty()); + let retained = store + .kv_get(1, b"busy:1", ReadMode::Linearizable as i32, 0, 2, 2) + .await; + assert!(retained.ok && !retained.not_found); +} diff --git a/lib/crowdb-kv/tests/store_test/multi_group_test.rs b/lib/crowdb-kv/tests/store_test/multi_group_test.rs index 9acd02f43..15ae6feca 100644 --- a/lib/crowdb-kv/tests/store_test/multi_group_test.rs +++ b/lib/crowdb-kv/tests/store_test/multi_group_test.rs @@ -68,6 +68,10 @@ async fn missing_group_returns_error() { // Put to non-existent group let resp = store.kv_put(99, b"k", b"v", 1, 1, 100, 1000).await; assert!(!resp.ok); + assert_eq!( + resp.error_code, + crowdb_kv::rpc::KvErrorCode::KvErrorNotLeader as i32 + ); assert!( resp.error.contains("no kv group"), "error should mention missing group, got: {}", diff --git a/lib/crowdb-kv/tests/wal_test.rs b/lib/crowdb-kv/tests/wal_test.rs index 6984e85fe..1c39a8e6b 100644 --- a/lib/crowdb-kv/tests/wal_test.rs +++ b/lib/crowdb-kv/tests/wal_test.rs @@ -18,6 +18,9 @@ mod wal_engine_test; #[path = "wal_test/replay_test.rs"] mod replay_test; +#[path = "wal_test/replay_compaction_test.rs"] +mod replay_compaction_test; + #[path = "wal_test/gc_test.rs"] mod gc_test; @@ -27,6 +30,9 @@ mod pipeline_backend_test; #[path = "wal_test/block_backend_test.rs"] mod block_backend_test; +#[path = "wal_test/block_runtime_test.rs"] +mod block_runtime_test; + #[path = "wal_test/block_restore_test.rs"] mod block_restore_test; diff --git a/lib/crowdb-kv/tests/wal_test/block_runtime_test.rs b/lib/crowdb-kv/tests/wal_test/block_runtime_test.rs new file mode 100644 index 000000000..2211f4829 --- /dev/null +++ b/lib/crowdb-kv/tests/wal_test/block_runtime_test.rs @@ -0,0 +1,141 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Real block-backed WAL I/O must leave a single Tokio worker responsive. + +use std::sync::{ + atomic::{AtomicBool, Ordering}, + mpsc, Arc, Barrier, +}; +use std::time::Duration; + +use crowdb_kv::paxos::roles::PxBallot; +use crowdb_kv::wal::{BlockDevice, IoBackend, OpenOptions, WALRecord, WalConfig, WalEngine}; + +async fn held_wal_io(sync: bool, expected_progress: bool) { + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("block-runtime-held"); + let device = BlockDevice::ssd(); + let backend = Arc::new(IoBackend::BlockDevice(device.clone())); + let wal = WalEngine::create( + Arc::clone(&backend), + WalConfig { + wal_disks: vec![directory.path().into()], + ..Default::default() + }, + 1, + ) + .await + .unwrap(); + // Finish initial segment creation before installing the one-operation gate. + wal.append(&WALRecord::from_promised(1, 1, 1, PxBallot::new(0, 1))) + .await + .unwrap(); + let entered = Arc::new(Barrier::new(2)); + let release = Arc::new(Barrier::new(2)); + device.set_io_gate_for_tests(sync, Arc::clone(&entered), Arc::clone(&release)); + let acknowledged = Arc::new(AtomicBool::new(false)); + let controller_acknowledged = Arc::clone(&acknowledged); + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let (progress_tx, progress_rx) = mpsc::channel(); + let controller = std::thread::spawn(move || { + entered.wait(); + let _ = started_tx.send(()); + let progressed = progress_rx.recv_timeout(Duration::from_secs(1)).is_ok(); + let premature_ack = controller_acknowledged.load(Ordering::Acquire); + release.wait(); + (progressed, premature_ack) + }); + let independent = tokio::spawn(async move { + started_rx.await.unwrap(); + let _ = progress_tx.send(()); + }); + wal.append(&WALRecord::from_promised(1, 1, 2, PxBallot::new(0, 1))) + .await + .unwrap(); + acknowledged.store(true, Ordering::Release); + independent.await.unwrap(); + let (progressed, premature_ack) = controller.join().unwrap(); + assert!( + !premature_ack, + "WAL acknowledged before the held durable I/O completed" + ); + assert_eq!( + progressed, expected_progress, + "held BlockDevice WAL I/O blocked the only Tokio worker" + ); + assert!(device.fdatasync_count() >= 2); + wal.flush_all().await.unwrap(); + drop(wal); + let recovered = crowdb_kv::wal::replay::replay_group(&backend, &[directory.path().into()], 1) + .await + .unwrap(); + assert_eq!( + recovered + .records + .iter() + .map(|record| record.slot) + .collect::>(), + vec![1, 2] + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn single_worker_progresses_while_block_wal_write_is_held() { + held_wal_io(false, true).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn single_worker_progresses_while_block_wal_sync_is_held() { + held_wal_io(true, true).await; +} + +#[tokio::test(flavor = "current_thread")] +async fn current_thread_fallback_remains_synchronous_without_panicking() { + held_wal_io(true, false).await; +} + +async fn block_io_round_trip() { + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("block-runtime-reopen"); + let path = directory.path().join("block"); + let backend = IoBackend::BlockDevice(BlockDevice::ssd()); + let mut file = backend.open(&path, OpenOptions::create_rw()).await.unwrap(); + file.write_at(b"first", 7).await.unwrap(); + file.write_vectored_at( + &[std::io::IoSlice::new(b"next"), std::io::IoSlice::new(b"last")], + 21, + ) + .await + .unwrap(); + file.fdatasync().await.unwrap(); + drop(file); + let mut reopened = backend.open(&path, OpenOptions::read_only()).await.unwrap(); + let mut first = [0; 5]; + reopened.read_exact_at(&mut first, 7).await.unwrap(); + let mut next = [0; 8]; + reopened.read_exact_at(&mut next, 21).await.unwrap(); + assert_eq!(&first, b"first"); + assert_eq!(&next, b"nextlast"); + let mut short = [0; 8]; + assert_eq!(reopened.read_at(&mut short, 4093).await.unwrap(), 3); + assert_eq!(&short[..3], &[0; 3]); + let mut beyond = [0; 1]; + assert_eq!( + reopened + .read_exact_at(&mut beyond, 8192) + .await + .unwrap_err() + .kind(), + std::io::ErrorKind::UnexpectedEof + ); + assert!(backend.open(&path, OpenOptions::create_new_rw()).await.is_err()); +} + +#[tokio::test(flavor = "current_thread")] +async fn current_thread_block_io_preserves_order_rmw_and_durable_reopen() { + block_io_round_trip().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn single_worker_block_io_preserves_order_rmw_and_durable_reopen() { + block_io_round_trip().await; +} diff --git a/lib/crowdb-kv/tests/wal_test/gc_test.rs b/lib/crowdb-kv/tests/wal_test/gc_test.rs index c44766dc3..ed7b2079c 100644 --- a/lib/crowdb-kv/tests/wal_test/gc_test.rs +++ b/lib/crowdb-kv/tests/wal_test/gc_test.rs @@ -17,6 +17,125 @@ fn sim_backend() -> Arc { Arc::new(IoBackend::MemBlock(MemBlockDevice::new())) } +#[tokio::test(flavor = "current_thread", start_paused = true)] +async fn gc_preserves_election_state_in_a_mixed_old_segment() { + let backend = sim_backend(); + let disks = vec![PathBuf::from("/mixed-vote")]; + let config = WalConfig { + wal_disks: disks.clone(), + wal_segment_size: 256, + ..Default::default() + }; + let wal = WalEngine::create(backend.clone(), config, 1).await.unwrap(); + wal.append(&WALRecord::from_vote_granted(1, 9, 42)).await.unwrap(); + for slot in 1..=20 { + let entry = PxLogEntry { + slot, + ballot: PxBallot::new(9, 42), + term: 9, + payload: Bytes::from_static(b"data"), + }; + wal.append(&WALRecord::from_accepted(1, &entry)).await.unwrap(); + } + wal.seal_all().await.unwrap(); + assert!(run_gc_with_watermark(&wal, 100).await.unwrap() > 0); + let replay = replay_group(&backend, &disks, 1).await.unwrap(); + assert_eq!(replay.current_term, 9); + assert_eq!( + replay.voted_for, + Some(42), + "GC must not permit a second vote after restart" + ); +} + +#[tokio::test(flavor = "current_thread", start_paused = true)] +async fn gc_keeps_a_high_term_without_a_vote_in_an_old_segment() { + let backend = sim_backend(); + let disks = vec![PathBuf::from("/mixed-term")]; + let config = WalConfig { + wal_disks: disks.clone(), + wal_segment_size: 256, + ..Default::default() + }; + let wal = WalEngine::create(backend.clone(), config, 1).await.unwrap(); + wal.append(&WALRecord::from_promised(1, 12, 1, PxBallot::new(12, 42))) + .await + .unwrap(); + // Older in-flight accepts can reach a different disk pipeline after a term change. + for slot in 2..=20 { + wal.append(&WALRecord::from_promised(1, 9, slot, PxBallot::new(9, 42))) + .await + .unwrap(); + } + wal.seal_all().await.unwrap(); + assert!(run_gc_with_watermark(&wal, 100).await.unwrap() > 0); + let replay = replay_group(&backend, &disks, 1).await.unwrap(); + assert_eq!(replay.current_term, 12); + assert_eq!(replay.voted_for, None); +} + +#[tokio::test(flavor = "current_thread", start_paused = true)] +async fn multi_disk_restart_preserves_index_and_election_state_across_gc() { + let backend = sim_backend(); + let disks = vec![PathBuf::from("/restart-a"), PathBuf::from("/restart-b")]; + let config = WalConfig { + wal_disks: disks.clone(), + wal_segment_size: 256, + ..Default::default() + }; + let mut wal = WalEngine::create(backend.clone(), config.clone(), 1) + .await + .unwrap(); + wal.append(&WALRecord::from_vote_granted(1, 9, 42)).await.unwrap(); + for round in 0..3 { + for slot in round * 24 + 1..=(round + 1) * 24 { + wal.append(&WALRecord::from_accepted( + 1, + &PxLogEntry { + slot, + ballot: PxBallot::new(9, 42), + term: 9, + payload: Bytes::copy_from_slice(&slot.to_le_bytes()), + }, + )) + .await + .unwrap(); + } + wal.seal_all().await.unwrap(); + let mut replay = replay_group(&backend, &disks, 1).await.unwrap(); + let prior_index = replay.index.clone(); + drop(wal); + wal = WalEngine::create_from_replay( + backend.clone(), + config.clone(), + 1, + replay.max_segment_id, + std::mem::take(&mut replay.index), + ) + .await + .unwrap(); + assert_eq!(wal.index().slot_count(), prior_index.slot_count()); + for record in &replay.records { + if record.slot != 0 { + assert_eq!( + wal.index().locate(record.slot).as_ref(), + prior_index.locate(record.slot) + ); + } + } + assert!(run_gc_with_watermark(&wal, (round + 1) * 24).await.unwrap() > 0); + let recovered = replay_group(&backend, &disks, 1).await.unwrap(); + assert_eq!(recovered.current_term, 9); + assert_eq!(recovered.voted_for, Some(42)); + let tip = (round + 1) * 24; + assert!(recovered + .records + .iter() + .any(|record| record.slot == tip && record.payload.as_ref() == tip.to_le_bytes())); + assert!(recovered.records.len() < replay.records.len()); + } +} + #[tokio::test(flavor = "current_thread", start_paused = true)] async fn gc_removes_segments_below_watermark() { let backend = sim_backend(); diff --git a/lib/crowdb-kv/tests/wal_test/index_test.rs b/lib/crowdb-kv/tests/wal_test/index_test.rs index b72365906..0b23c488d 100644 --- a/lib/crowdb-kv/tests/wal_test/index_test.rs +++ b/lib/crowdb-kv/tests/wal_test/index_test.rs @@ -25,6 +25,8 @@ fn meta(seg: u64, min: u64, max: u64) -> SegmentMeta { min_slot: min, max_slot: max, record_count: (max - min + 1) as u32, + max_term: 0, + vote_term: None, } } diff --git a/lib/crowdb-kv/tests/wal_test/record_test.rs b/lib/crowdb-kv/tests/wal_test/record_test.rs index 9bdd1ab3e..634c855c5 100644 --- a/lib/crowdb-kv/tests/wal_test/record_test.rs +++ b/lib/crowdb-kv/tests/wal_test/record_test.rs @@ -7,6 +7,15 @@ use bytes::Bytes; use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; use crowdb_kv::wal::record::{RecordError, RecordType, WALRecord, MIN_RECORD_SIZE}; +#[test] +fn short_frame_marker_is_rejected() { + for frame_len in 0_u32..55 { + let mut frame = vec![0; 128]; + frame[..4].copy_from_slice(&frame_len.to_le_bytes()); + assert!(matches!(WALRecord::decode(&frame), Err(RecordError::Truncated))); + } +} + #[test] fn encode_decode_roundtrip_accepted() { let entry = PxLogEntry { diff --git a/lib/crowdb-kv/tests/wal_test/replay_compaction_test.rs b/lib/crowdb-kv/tests/wal_test/replay_compaction_test.rs new file mode 100644 index 000000000..6f87b83db --- /dev/null +++ b/lib/crowdb-kv/tests/wal_test/replay_compaction_test.rs @@ -0,0 +1,116 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::path::PathBuf; +use std::sync::Arc; + +use bytes::Bytes; +use crowdb_kv::cluster::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_kv::wal::record::{RecordType, WALRecord}; +use crowdb_kv::wal::replay::replay_group; +use crowdb_kv::wal::{IoBackend, MemBlockDevice, WalConfig, WalEngine}; + +fn accepted(ballot: u64, term: u64, value: &[u8]) -> WALRecord { + let mut payload = vec![1, 0, 0, 1, 0, 0, 0, b'k']; + payload.extend_from_slice(&u32::try_from(value.len()).unwrap().to_le_bytes()); + payload.extend_from_slice(value); + WALRecord::from_accepted( + 1, + &PxLogEntry { + slot: 1, + ballot: PxBallot::new(ballot, 7), + term, + payload: Bytes::from(payload), + }, + ) +} + +async fn fixture() -> (Arc, WalConfig, Arc) { + let backend = Arc::new(IoBackend::MemBlock(MemBlockDevice::new())); + let config = WalConfig { + wal_disks: vec![PathBuf::from("/compact")], + ..Default::default() + }; + let wal = WalEngine::create(backend.clone(), config.clone(), 1) + .await + .unwrap(); + (backend, config, wal) +} + +#[tokio::test(flavor = "current_thread", start_paused = true)] +async fn compaction_preserves_accepted_value_below_an_independent_higher_promise() { + let (backend, config, wal) = fixture().await; + let highest = accepted(10, 10, b"highest"); + for record in [ + WALRecord::from_promised(1, 20, 1, PxBallot::new(20, 7)), + accepted(4, 4, b"old"), + highest.clone(), + WALRecord::from_promised(1, 12, 1, PxBallot::new(12, 7)), + accepted(5, 5, b"later-old"), + ] { + wal.append(&record).await.unwrap(); + } + wal.seal_all().await.unwrap(); + let replay = replay_group(&backend, &config.wal_disks, 1).await.unwrap(); + assert_eq!(replay.records.len(), 2); + assert_eq!(replay.current_term, 20); + assert_eq!( + replay + .index + .segments() + .map(|segment| segment.record_count) + .sum::(), + 5 + ); + let replica = PxLocalReplica::restore_from_replay(7, PxLocalReplicaRole::Follower, &replay) + .await + .unwrap(); + assert_eq!(replica.accepted_at(1).await, highest.to_log_entry()); + assert_eq!(replica.promised_at(1).await, Some(PxBallot::new(20, 7))); + assert_eq!( + replica.learner.engine_get(b"k").await, + Some((1, b"highest".to_vec())) + ); +} + +#[tokio::test(flavor = "current_thread", start_paused = true)] +async fn discarded_accepts_still_advance_the_term_and_fence_old_votes() { + let (backend, config, wal) = fixture().await; + for record in [ + accepted(10, 10, b"highest"), + accepted(4, 30, b"old"), + WALRecord::from_vote_granted(1, 30, 42), + WALRecord::from_vote_granted(1, 12, 99), + ] { + wal.append(&record).await.unwrap(); + } + wal.seal_all().await.unwrap(); + let replay = replay_group(&backend, &config.wal_disks, 1).await.unwrap(); + assert_eq!(replay.current_term, 30); + assert_eq!(replay.voted_for, Some(42)); + assert_eq!( + replay + .records + .iter() + .filter(|record| record.record_type == RecordType::Accepted) + .count(), + 1 + ); + wal.append(&accepted(3, 31, b"discarded-new-term")).await.unwrap(); + wal.seal_all().await.unwrap(); + let replay = replay_group(&backend, &config.wal_disks, 1).await.unwrap(); + assert_eq!(replay.current_term, 31); + assert_eq!(replay.voted_for, None); +} + +#[tokio::test(flavor = "current_thread", start_paused = true)] +async fn equal_ballot_compaction_keeps_the_last_verified_accept() { + let (backend, config, wal) = fixture().await; + wal.append(&accepted(5, 5, b"earlier")).await.unwrap(); + let latest = accepted(5, 5, b"latest"); + wal.append(&latest).await.unwrap(); + wal.seal_all().await.unwrap(); + let replay = replay_group(&backend, &config.wal_disks, 1).await.unwrap(); + assert_eq!(replay.records, vec![latest]); +} diff --git a/lib/crowdb-kv/tests/wal_test/replay_test.rs b/lib/crowdb-kv/tests/wal_test/replay_test.rs index bb0d527ab..dafaf7c79 100644 --- a/lib/crowdb-kv/tests/wal_test/replay_test.rs +++ b/lib/crowdb-kv/tests/wal_test/replay_test.rs @@ -382,16 +382,63 @@ async fn restore_from_replay_with_engine_resumes_from_last_applied_slot() { assert!(restored.accepted_at(3).await.is_some()); } -/// Defensive fallback: if the engine's resume floor doesn't line up with an -/// accepted entry in the WAL-rebuilt acceptor (not expected in practice -- -/// an engine can only durably apply a slot that was itself accepted and -/// WAL-logged -- but not a correctness invariant this restore path should -/// ever trust blindly), restore still skips straight to `resume_from + 1` -/// (never re-attempts the skipped prefix -- see the "no safe fallback" -/// rationale on `restore_from_replay_with_engine`) but leaves the frontier -/// at its conservative fresh-learner default instead of guessing a term. +#[tokio::test] +async fn durable_snapshot_prefix_survives_reclaimed_wal_boundary_records() { + let tmp = crowdb_test_harness::test_dirs::tempdir_in_test_data("snapshot-reclaimed-boundary"); + let options = CrowdbTreeConfig { + path: Some(tmp.path().to_string_lossy().into_owned()), + ..Default::default() + }; + let engine = CrowdbTreeEngine::open(&options).unwrap(); + for slot in 1..=3 { + engine + .apply( + slot, + &Batch::decode(&Bytes::from(encode_put_payload( + b"durable", + &[u8::try_from(slot).unwrap()], + ))), + ) + .into_ready() + .unwrap(); + } + engine.flush(); + assert_eq!(engine.persist_snapshot(), 3); + drop(engine); + let backend = sim_backend(); + let disks = vec![PathBuf::from("/reclaimed")]; + let wal = WalEngine::create(backend.clone(), test_config(&disks), 1) + .await + .unwrap(); + wal.seal_all().await.unwrap(); + let replay = replay_group(&backend, &disks, 1).await.unwrap(); + assert!(replay.records.is_empty()); + let restored = PxLocalReplica::restore_from_replay_with_engine( + 7, + PxLocalReplicaRole::Follower, + &replay, + Box::new(CrowdbTreeEngine::open(&options).unwrap()), + ) + .await + .unwrap(); + assert_eq!(restored.learner.engine_get(b"durable").await, Some((3, vec![3]))); + assert_eq!( + restored.contiguous_applied(), + 3, + "snapshot prefix must not become an unfillable WAL gap" + ); + assert_eq!(restored.contiguous_chosen(), 3); + assert_eq!( + restored.last_chosen_term(), + 0, + "a reclaimed boundary has an unknown historical term" + ); +} + +/// The durable engine owns its covered prefix even when the retained WAL +/// omits the boundary. Later entries must extend that prefix normally. #[tokio::test(flavor = "current_thread", start_paused = true)] -async fn restore_from_replay_with_engine_falls_back_when_resume_slot_has_no_accepted_entry() { +async fn restore_from_replay_with_engine_keeps_prefix_when_resume_slot_has_no_accepted_entry() { let backend = sim_backend(); let disks = vec![PathBuf::from("/wal")]; let config = test_config(&disks); @@ -414,22 +461,15 @@ async fn restore_from_replay_with_engine_falls_back_when_resume_slot_has_no_acce // accepted entry for. `last_applied_slot()` is itself a contiguous // watermark (crowdb-tree folds `received_slots_` forward from its own // frontier), so reaching floor 2 legitimately requires applying slot 1 - // too -- modeling an engine that durably has extra data at slot 2 with - // no independent WAL corroboration (e.g. a lost/truncated WAL record), - // rather than an outright-impossible-via-the-API state. + // too. The retained WAL may omit the already durable boundary after + // reclamation; it need not corroborate that complete prefix again. let engine = CrowdbTreeEngine::open(&CrowdbTreeConfig::default()).expect("open crowdb-tree engine"); engine - .apply( - 1, - &Batch::decode(&Bytes::from(encode_put_payload(b"phantom1", b"x"))), - ) + .apply(1, &Batch::decode(&Bytes::from(encode_put_payload(b"k1", b"v1")))) .into_ready() .unwrap(); engine - .apply( - 2, - &Batch::decode(&Bytes::from(encode_put_payload(b"phantom2", b"y"))), - ) + .apply(2, &Batch::decode(&Bytes::from(encode_put_payload(b"k2", b"v2")))) .into_ready() .unwrap(); engine.handle().flush().expect("flush"); @@ -448,21 +488,22 @@ async fn restore_from_replay_with_engine_falls_back_when_resume_slot_has_no_acce .await .unwrap(); - // Slots 1 and 2 are skipped outright (never re-attempted -- see the - // rationale above), so slot 1's WAL value is *not* recovered here: an - // honest, safe degradation for this assumed-impossible mismatch, not a - // silent correctness violation (nothing that was ever safely writable - // is lost or corrupted). Slot 3, past the skipped prefix, replays - // normally. - assert_eq!(restored.learner.engine_get(b"k1").await.map(|(_, v)| v), None); + // Slots 1 and 2 retain the engine's complete prefix. Slot 3 extends it + // normally without needing the reclaimed boundary record. + assert_eq!( + restored.learner.engine_get(b"k1").await.map(|(_, v)| v), + Some(b"v1".to_vec()) + ); + assert_eq!( + restored.learner.engine_get(b"k2").await.map(|(_, v)| v), + Some(b"v2".to_vec()) + ); assert_eq!( restored.learner.engine_get(b"k3").await.map(|(_, v)| v), Some(b"v3".to_vec()) ); - // No accepted entry at the resume floor (slot 2) to seed a term from, so - // the frontier stays at the fresh-learner default until slot 3's - // out-of-order `learn()` bumps the max-ever-seen high-water mark. - assert_eq!(restored.contiguous_chosen(), 0); + assert_eq!(restored.contiguous_chosen(), 3); + assert_eq!(restored.contiguous_applied(), 3); assert_eq!(restored.last_chosen_slot(), 3); assert_eq!(restored.last_chosen_term(), 1); } diff --git a/lib/crowdb-kv/tests/wal_test/segment_test.rs b/lib/crowdb-kv/tests/wal_test/segment_test.rs index d190b15af..b55f57dbb 100644 --- a/lib/crowdb-kv/tests/wal_test/segment_test.rs +++ b/lib/crowdb-kv/tests/wal_test/segment_test.rs @@ -38,6 +38,35 @@ fn accepted_record(group: u64, slot: u64, payload_len: usize) -> WALRecord { } } +#[tokio::test(flavor = "current_thread", start_paused = true)] +async fn buffered_binary_replay_keeps_payloads_across_refills_and_large_frames() { + let backend = aligned_backend(); + let directory = PathBuf::from("/buffer-boundary/group1"); + let mut segment = WalSegment::create(&backend, &directory, 1, 1).await.unwrap(); + let mut records = Vec::new(); + for (index, length) in [13, 65536, 1_048_500, 37, 1_048_577, 0, 32768] + .into_iter() + .enumerate() + { + let record = accepted_record(1, u64::try_from(index).unwrap() + 1, length); + segment.append(&record).await.unwrap(); + records.push(record); + } + segment.seal().await.unwrap(); + let mut reader = SegmentReader::open(&backend, segment.path()).await.unwrap(); + let mut recovered = Vec::new(); + while let Some((record, _)) = reader.next_record().await.unwrap() { + recovered.push(record); + } + assert!(reader.read_footer().await.unwrap().is_some()); + drop(reader); + drop(segment); + assert_eq!( + recovered, records, + "earlier payload owners must survive every refill" + ); +} + #[tokio::test(flavor = "current_thread", start_paused = true)] async fn create_segment_writes_header() { let backend = sim_backend(); diff --git a/lib/crowdb-protocol/src/mgmt/node/deployment.rs b/lib/crowdb-protocol/src/mgmt/node/deployment.rs index c4a289dc0..32f6d460f 100644 --- a/lib/crowdb-protocol/src/mgmt/node/deployment.rs +++ b/lib/crowdb-protocol/src/mgmt/node/deployment.rs @@ -4,6 +4,12 @@ //! Accepted startup inputs, distinct from discovered or confirmed membership. use super::setup::{NodeSetupChoices, SetupService, SetupServiceKind, SetupStorage}; +use crate::{ + ACCESS_SERVER_DATASET_HTTP_BASE, ACCESS_SERVER_HEALTH_HTTP_BASE, ACCESS_SERVER_HTTP_BASE, + ACCESS_SERVER_ICEBERG_HTTP_BASE, CHUNKDB_HTTP_BASE, CHUNKDB_RPC_BASE, CHUNK_KV_HTTP_BASE, + CHUNK_KV_RPC_BASE, DISKDB_HTTP_BASE, DISKDB_LISTEN_BASE, DISKDB_RPC_BASE, DISKIO_RPC_BASE, + KV_SERVER_LISTEN_BASE, KV_SERVER_MGMT_BASE, +}; use serde::{Deserialize, Serialize}; mod public_access; @@ -30,7 +36,7 @@ impl FormationPolicy { /// # Errors /// Rejects unsupported versions, malformed scopes and voter counts. pub fn validate(&self) -> Result<(), &'static str> { - if self.version != 1 || !matches!(self.initial_voters, 1 | 3) || !valid_cluster(&self.cluster) { + if self.version != 1 || !matches!(self.initial_voters, 1 | 3 | 5) || !valid_cluster(&self.cluster) { return Err("invalid node formation policy"); } Ok(()) @@ -73,24 +79,28 @@ impl Default for NodeDeploymentPolicy { let listeners = &mut service.listeners; match service.kind { SetupServiceKind::PaxosKv => { - listeners.http_port = Some(10000); - listeners.rpc_port = Some(10100); + listeners.http_port = Some(KV_SERVER_MGMT_BASE); + listeners.rpc_port = Some(KV_SERVER_LISTEN_BASE); } - SetupServiceKind::Diskdb => listeners.rpc_port = Some(7100), - SetupServiceKind::Diskio => listeners.rpc_port = Some(7200), + SetupServiceKind::Diskdb => { + listeners.listen_port = Some(DISKDB_LISTEN_BASE); + listeners.http_port = Some(DISKDB_HTTP_BASE); + listeners.rpc_port = Some(DISKDB_RPC_BASE); + } + SetupServiceKind::Diskio => listeners.rpc_port = Some(DISKIO_RPC_BASE), SetupServiceKind::Chunkdb => { - listeners.http_port = Some(7300); - listeners.rpc_port = Some(7301); + listeners.http_port = Some(CHUNKDB_HTTP_BASE); + listeners.rpc_port = Some(CHUNKDB_RPC_BASE); } SetupServiceKind::ChunkKv => { - listeners.http_port = Some(7400); - listeners.rpc_port = Some(7401); + listeners.http_port = Some(CHUNK_KV_HTTP_BASE); + listeners.rpc_port = Some(CHUNK_KV_RPC_BASE); } SetupServiceKind::AccessServer => { - listeners.s3_port = Some(9091); - listeners.http_port = Some(9092); - listeners.dataset_port = Some(9093); - listeners.health_port = Some(9094); + listeners.s3_port = Some(ACCESS_SERVER_HTTP_BASE); + listeners.http_port = Some(ACCESS_SERVER_ICEBERG_HTTP_BASE); + listeners.dataset_port = Some(ACCESS_SERVER_DATASET_HTTP_BASE); + listeners.health_port = Some(ACCESS_SERVER_HEALTH_HTTP_BASE); } } } @@ -101,9 +111,9 @@ impl Default for NodeDeploymentPolicy { initial_voters: 1, setup, public_access: PublicAccessUris { - s3: Some("http://127.0.0.1:9091".into()), - iceberg: Some("http://127.0.0.1:9092".into()), - dataset: Some("http://127.0.0.1:9093".into()), + s3: Some(format!("http://127.0.0.1:{ACCESS_SERVER_HTTP_BASE}")), + iceberg: Some(format!("http://127.0.0.1:{ACCESS_SERVER_ICEBERG_HTTP_BASE}")), + dataset: Some(format!("http://127.0.0.1:{ACCESS_SERVER_DATASET_HTTP_BASE}")), }, } } @@ -128,8 +138,8 @@ impl NodeDeploymentPolicy { if self.version != 1 { return Err("unsupported node deployment policy version"); } - if !matches!(self.initial_voters, 1 | 3) { - return Err("initial voters must be one or three"); + if !matches!(self.initial_voters, 1 | 3 | 5) { + return Err("initial voters must be one, three or five"); } if !valid_cluster(&self.cluster) { return Err("cluster scope must contain 1 to 48 name characters"); @@ -141,12 +151,8 @@ impl NodeDeploymentPolicy { } for service in self.setup.services.iter().filter(|service| service.enabled) { let ports = &service.listeners; - if service.kind == SetupServiceKind::PaxosKv - && (ports.http_port != Some(10000) || ports.rpc_port != Some(10100)) - { - return Err("bootstrap KV requires management port 10000 and RPC port 10100"); - } if [ + ports.listen_port, ports.http_port, ports.rpc_port, ports.s3_port, @@ -155,7 +161,7 @@ impl NodeDeploymentPolicy { ] .into_iter() .flatten() - .any(|port| matches!(port, 2222 | 9090 | 9095 | 10101)) + .any(|port| matches!(port, 2222 | 9090 | 9095)) { return Err("service listener conflicts with node management or bootstrap transport"); } diff --git a/lib/crowdb-protocol/src/mgmt/node/setup.rs b/lib/crowdb-protocol/src/mgmt/node/setup.rs index ab378730c..847c0de49 100644 --- a/lib/crowdb-protocol/src/mgmt/node/setup.rs +++ b/lib/crowdb-protocol/src/mgmt/node/setup.rs @@ -33,6 +33,8 @@ impl SetupServiceKind { #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct SetupListeners { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub listen_port: Option, pub http_port: Option, pub rpc_port: Option, pub s3_port: Option, @@ -117,6 +119,31 @@ pub struct NodeSetupChoices { pub services: Vec, } +impl NodeSetupChoices { + /// Whether the selected storage or services require ordinary metadata. + #[must_use] + pub fn requires_default_preparation(&self) -> bool { + !matches!(self.storage, SetupStorage::Diskless) + || self + .services + .iter() + .any(|service| service.enabled && service.kind != SetupServiceKind::PaxosKv) + } + + /// Return the management and RPC listeners reserved for the bootstrap KV. + /// + /// The node launcher may shift these listeners for host-network containers + /// that share one physical host. Callers that need the Group 0 endpoint + /// must use the accepted policy rather than assuming the default ports. + #[must_use] + pub fn paxos_kv_ports(&self) -> Option<(u16, u16)> { + self.services + .iter() + .find(|service| service.kind == SetupServiceKind::PaxosKv && service.enabled) + .and_then(|service| Some((service.listeners.http_port?, service.listeners.rpc_port?))) + } +} + fn default_services() -> Vec { SetupServiceKind::ALL .into_iter() diff --git a/lib/crowdb-protocol/src/mgmt/node/setup/validation.rs b/lib/crowdb-protocol/src/mgmt/node/setup/validation.rs index c312f2e7a..4de75f2d7 100644 --- a/lib/crowdb-protocol/src/mgmt/node/setup/validation.rs +++ b/lib/crowdb-protocol/src/mgmt/node/setup/validation.rs @@ -137,12 +137,6 @@ impl NodeSetupChoices { fn valid_listeners(service: &SetupService, ports: &mut BTreeSet) -> bool { use SetupServiceKind::{AccessServer, Chunkdb, Diskdb, Diskio}; let listeners = &service.listeners; - if service.kind == SetupServiceKind::PaxosKv - && (listeners.http_port.is_some_and(|port| port != 10000) - || listeners.rpc_port.is_some_and(|port| port != 10100)) - { - return false; - } if (service.kind != Chunkdb && listeners.dynamic_ownership.is_some()) || (service.kind != Diskdb && listeners.keepalive_interval_secs.is_some()) || (service.kind != AccessServer @@ -150,7 +144,11 @@ fn valid_listeners(service: &SetupService, ports: &mut BTreeSet) -> bool { || listeners.health_port.is_some() || listeners.dataset_port.is_some())) || (service.kind == AccessServer && listeners.rpc_port.is_some()) - || (matches!(service.kind, Diskdb | Diskio) && listeners.http_port.is_some()) + || (service.kind != Diskdb && listeners.listen_port.is_some()) + || (service.kind == Diskio && listeners.http_port.is_some()) + || (service.kind == Diskdb + && (listeners.listen_port.is_some() != listeners.http_port.is_some() + || (listeners.listen_port.is_some() && listeners.rpc_port.is_none()))) { return false; } @@ -158,6 +156,7 @@ fn valid_listeners(service: &SetupService, ports: &mut BTreeSet) -> bool { return true; } for port in [ + listeners.listen_port, listeners.http_port, listeners.rpc_port, listeners.s3_port, @@ -171,7 +170,7 @@ fn valid_listeners(service: &SetupService, ports: &mut BTreeSet) -> bool { return false; } } - if service.kind == Diskdb { + if service.kind == Diskdb && listeners.listen_port.is_none() { if let Some(port) = listeners.rpc_port { return port <= 65533 && ports.insert(port + 1) && ports.insert(port + 2); } diff --git a/lib/crowdb-protocol/tests/chunk_task_value_test.rs b/lib/crowdb-protocol/tests/chunk_task_value_test.rs index fad1ccb4f..2b72b9e81 100644 --- a/lib/crowdb-protocol/tests/chunk_task_value_test.rs +++ b/lib/crowdb-protocol/tests/chunk_task_value_test.rs @@ -50,7 +50,7 @@ fn task_value_rejects_wrong_identifier_and_malformed_buffer() { #[test] fn task_value_bounds_last_error_without_splitting_utf8() { let mut value = task_value(); - value.last_error = "界".repeat(400); + value.last_error = "\u{754c}".repeat(400); let decoded = decode_chunk_task_value(&encode_chunk_task_value(&value)).unwrap(); assert!(decoded.last_error.len() <= 1_024); assert!(decoded.last_error.is_char_boundary(decoded.last_error.len())); diff --git a/lib/crowdb-protocol/tests/node_deployment_test.rs b/lib/crowdb-protocol/tests/node_deployment_test.rs index 86194b55f..e2a70df6b 100644 --- a/lib/crowdb-protocol/tests/node_deployment_test.rs +++ b/lib/crowdb-protocol/tests/node_deployment_test.rs @@ -3,7 +3,10 @@ use crowdb_protocol::mgmt::node::{ deployment::{DeploymentMode, NodeDeploymentOverrides, NodeDeploymentPolicy, PublicAccessUris}, - setup::{SetupServiceKind, SetupStorage}, + setup::{NodeSetupChoices, SetupServiceKind, SetupStorage}, +}; +use crowdb_protocol::{ + ACCESS_SERVER_DATASET_HTTP_BASE, ACCESS_SERVER_HTTP_BASE, ACCESS_SERVER_ICEBERG_HTTP_BASE, }; #[test] @@ -19,15 +22,22 @@ fn fresh_local_policy_has_one_voter_and_all_public_access_listeners() { SetupStorage::Automatic { sparse: true, .. } )); assert!(policy.setup.services.iter().all(|service| service.enabled)); + let diskdb = &policy.setup.services[1].listeners; + assert_eq!(diskdb.listen_port, Some(11000)); + assert_eq!(diskdb.http_port, Some(11100)); + assert_eq!(diskdb.rpc_port, Some(11200)); let access = policy .setup .services .iter() .find(|service| service.kind == SetupServiceKind::AccessServer) .unwrap(); - assert_eq!(access.listeners.s3_port, Some(9091)); - assert_eq!(access.listeners.http_port, Some(9092)); - assert_eq!(access.listeners.dataset_port, Some(9093)); + assert_eq!(access.listeners.s3_port, Some(ACCESS_SERVER_HTTP_BASE)); + assert_eq!(access.listeners.http_port, Some(ACCESS_SERVER_ICEBERG_HTTP_BASE)); + assert_eq!( + access.listeners.dataset_port, + Some(ACCESS_SERVER_DATASET_HTTP_BASE) + ); } #[test] @@ -60,6 +70,19 @@ fn public_origins_resolve_per_field_and_survive_omitted_restart_inputs() { assert_eq!(accepted.setup, NodeDeploymentPolicy::default().setup); } +#[test] +fn shifted_setup_retains_the_group0_listener_pair_for_admission_metadata() { + let mut choices = NodeSetupChoices::default(); + let kv = choices + .services + .iter_mut() + .find(|service| service.kind == SetupServiceKind::PaxosKv) + .unwrap(); + kv.listeners.http_port = Some(10001); + kv.listeners.rpc_port = Some(10101); + assert_eq!(choices.paxos_kv_ports(), Some((10001, 10101))); +} + #[test] fn malformed_or_secret_bearing_public_origins_never_resolve() { for uri in [ @@ -175,16 +198,16 @@ fn invalid_policies_are_rejected_before_any_resolution_can_replace_them() { .is_err()); } let mut duplicate_port = valid; - duplicate_port.setup.services[5].listeners.dataset_port = Some(9092); + duplicate_port.setup.services[5].listeners.dataset_port = Some(ACCESS_SERVER_ICEBERG_HTTP_BASE); assert!(duplicate_port.validate().is_err()); duplicate_port.setup.services[5].listeners.dataset_port = None; assert!(duplicate_port.validate().is_err()); - for reserved in [2222, 9090, 9095, 10101] { + for reserved in [2222, 9090, 9095] { let mut policy = NodeDeploymentPolicy::default(); policy.setup.services[5].listeners.dataset_port = Some(reserved); assert!(policy.validate().is_err()); } - for (http, rpc) in [(10001, 10100), (10000, 10102)] { + for (http, rpc) in [(0, 10100), (10000, 10000)] { let mut policy = NodeDeploymentPolicy::default(); let kv = policy .setup @@ -194,7 +217,7 @@ fn invalid_policies_are_rejected_before_any_resolution_can_replace_them() { .unwrap(); kv.listeners.http_port = Some(http); kv.listeners.rpc_port = Some(rpc); - assert!(policy.validate().unwrap_err().contains("bootstrap KV")); + assert!(policy.validate().is_err()); } } @@ -220,3 +243,29 @@ fn accepted_serialization_requires_explicit_version_and_complete_policy() { }; assert!(current.resolve(&future, None).is_err()); } + +#[test] +fn five_voter_policy_retains_the_accepted_formation_and_rejects_other_counts() { + let omitted = NodeDeploymentOverrides::default(); + let explicit = NodeDeploymentOverrides { + initial_voters: Some(5), + ..Default::default() + }; + let accepted = explicit.resolve(&omitted, None).unwrap(); + accepted.formation().validate().unwrap(); + assert_eq!(accepted.initial_voters, 5); + assert_eq!(omitted.resolve(&omitted, Some(&accepted)).unwrap(), accepted); + let changed = NodeDeploymentOverrides { + initial_voters: Some(3), + ..Default::default() + }; + assert!(changed.resolve(&omitted, Some(&accepted)).is_err()); + for count in [0, 2, 4, 6, 7, 255] { + let invalid = NodeDeploymentPolicy { + initial_voters: count, + ..Default::default() + }; + assert!(invalid.validate().is_err(), "count {count}"); + assert!(invalid.formation().validate().is_err(), "count {count}"); + } +} diff --git a/lib/crowdb-protocol/tests/node_setup_test.rs b/lib/crowdb-protocol/tests/node_setup_test.rs index a95236efb..cf2bbbe2d 100644 --- a/lib/crowdb-protocol/tests/node_setup_test.rs +++ b/lib/crowdb-protocol/tests/node_setup_test.rs @@ -117,15 +117,11 @@ fn service_selection_and_reserved_diskdb_listener_range_cannot_be_ambiguous() { let mut unsupported = request(); unsupported.choices.services[0].listeners.s3_port = Some(32000); assert!(unsupported.validate().is_err()); - let mut fixed_bootstrap = request(); - fixed_bootstrap.choices.services[0].listeners.http_port = Some(19910); - assert!( - fixed_bootstrap.validate().is_err(), - "node setup cannot silently ignore a bootstrap port override" - ); - fixed_bootstrap.choices.services[0].listeners.http_port = Some(10000); - fixed_bootstrap.choices.services[0].listeners.rpc_port = Some(10100); - fixed_bootstrap.validate().unwrap(); + let mut shifted_bootstrap = request(); + shifted_bootstrap.choices.services[0].listeners.http_port = Some(10100); + shifted_bootstrap.choices.services[0].listeners.rpc_port = Some(10200); + shifted_bootstrap.validate().unwrap(); + assert_eq!(shifted_bootstrap.choices.paxos_kv_ports(), Some((10100, 10200))); } #[test] diff --git a/lib/crowdb-rpc/ffi/src/logging.rs b/lib/crowdb-rpc/ffi/src/logging.rs index 573ab583e..8c428df6e 100644 --- a/lib/crowdb-rpc/ffi/src/logging.rs +++ b/lib/crowdb-rpc/ffi/src/logging.rs @@ -51,6 +51,7 @@ pub fn add_log_stderr(level: &str) { /// Flush and stop the C++ spdlog logger. Call at process exit. pub fn shutdown_logging() { + crate::owned_route::drain_before_native_shutdown(); unsafe { sys::crowdb_rpc_shutdown_logging() }; } diff --git a/lib/crowdb-rpc/ffi/src/owned_route.rs b/lib/crowdb-rpc/ffi/src/owned_route.rs index 7f33c94ae..d6c9f73d6 100644 --- a/lib/crowdb-rpc/ffi/src/owned_route.rs +++ b/lib/crowdb-rpc/ffi/src/owned_route.rs @@ -8,6 +8,10 @@ use std::sync::Arc; use crate::{Connection, RpcClient, RpcServer}; +mod retirement; + +pub(crate) use retirement::shutdown as drain_before_native_shutdown; + /// Supplies the raw crowdb-rpc client handle while an owner keeps it alive. pub trait RpcClientHandle: Send + Sync { #[doc(hidden)] @@ -23,19 +27,41 @@ impl RpcClientHandle for RpcClient { /// A matching client, server, and connection whose owners travel together. /// /// This is the safe handoff boundary for higher-level FFIs that retain raw -/// crowdb-rpc handles after their constructor returns. +/// crowdb-rpc handles after their constructor returns. Clones share one owner +/// bundle. Its final release schedules native teardown outside RPC callbacks; +/// those callbacks must return without waiting for that teardown. #[derive(Clone)] pub struct OwnedClientRoute { + handles: Arc, +} + +struct RouteHandles { + owners: Option, +} + +struct RouteOwners { client: Arc, server: Arc, connection: Connection, } +impl Drop for RouteHandles { + fn drop(&mut self) { + if let Some(owners) = self.owners.take() { + retirement::retire(owners); + } + } +} + impl OwnedClientRoute { + fn owners(&self) -> &RouteOwners { + self.handles.owners.as_ref().expect("live route owners") + } + /// Whether this retained connection can still accept requests. #[must_use] pub fn is_open(&self) -> bool { - self.connection.is_open() + self.owners().connection.is_open() } #[must_use] @@ -43,22 +69,52 @@ impl OwnedClientRoute { where C: RpcClientHandle + 'static, { + // Initialize the lifecycle worker before publishing a route. Ordinary + // request-path clones and releases only change this Arc's refcount. + retirement::register_owner(); Self { - client, - server, - connection, + handles: Arc::new(RouteHandles { + owners: Some(RouteOwners { + client, + server, + connection, + }), + }), } } #[doc(hidden)] #[must_use] pub fn raw_handles(&self) -> (*mut c_void, *mut c_void, *mut c_void) { + let owners = self.owners(); ( - self.client.rpc_client_handle(), - self.server.handle().cast(), - self.connection.handle().cast(), + owners.client.rpc_client_handle(), + owners.server.handle().cast(), + owners.connection.handle().cast(), ) } + + /// Wait for queued and recursively queued owner teardown after route + /// creators and callbacks have quiesced. Never call from an RPC callback. + #[cfg(feature = "test-util")] + #[must_use] + pub fn drain_retired_for_tests(timeout: std::time::Duration) -> bool { + retirement::drain(timeout) + } + + /// Number of owner bundles queued or being destroyed on the lifecycle worker. + #[cfg(feature = "test-util")] + #[must_use] + pub fn pending_retired_for_tests() -> usize { + retirement::pending() + } + + /// Finish queued native teardown after quiescing route creators and + /// callbacks and dropping their routes. Never call from an RPC callback. + #[must_use] + pub fn drain_retired(timeout: std::time::Duration) -> bool { + retirement::drain(timeout) && retirement::quiescent() + } } impl std::fmt::Debug for OwnedClientRoute { diff --git a/lib/crowdb-rpc/ffi/src/owned_route/retirement.rs b/lib/crowdb-rpc/ffi/src/owned_route/retirement.rs new file mode 100644 index 000000000..725555434 --- /dev/null +++ b/lib/crowdb-rpc/ffi/src/owned_route/retirement.rs @@ -0,0 +1,147 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Process-lifetime owner teardown, independent of Tokio runtime shutdown. +//! +//! Only the last clone of a route wrapper queues work. In-flight RPC lease +//! clones never use the channel. Teardown can synchronously complete other +//! RPCs, so neither producers nor the worker may wait for queue capacity. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{mpsc, Arc, OnceLock}; +use std::time::Instant; + +use crowdb_common::metrics::{self, Counter, Gauge, LatencyHistogram}; + +use super::RouteOwners; + +unsafe extern "C" { + fn atexit(callback: extern "C" fn()) -> std::ffi::c_int; +} + +static WORKER: OnceLock = OnceLock::new(); + +struct Task { + owners: RouteOwners, + queued_at: Instant, +} + +struct State { + live: AtomicUsize, + pending: AtomicUsize, + pending_metric: Arc, + completed: Arc, + latency: Arc, +} + +struct Retirement { + sender: mpsc::Sender, + state: Arc, +} + +fn worker() -> &'static Retirement { + WORKER.get_or_init(|| { + let (sender, receiver) = mpsc::channel::(); + let state = Arc::new(State { + live: AtomicUsize::new(0), + pending: AtomicUsize::new(0), + pending_metric: metrics::global_gauge("rpc.route.retirement.pending"), + completed: metrics::global_counter("rpc.route.retirement.completed"), + latency: metrics::global_histogram("rpc.route.retirement.lh"), + }); + let worker_state = Arc::clone(&state); + // The static sender remains admitted through the last resolver/callback + // release. No runtime shutdown or JoinHandle drop can close it early. + std::thread::Builder::new() + .name("rpc-route-retire".to_owned()) + .spawn(move || { + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + for task in receiver { + drop(task.owners); + worker_state + .latency + .observe(task.queued_at.elapsed().as_nanos().try_into().unwrap_or(u64::MAX)); + worker_state.completed.inc(); + worker_state.pending_metric.dec(); + worker_state.live.fetch_sub(1, Ordering::Release); + // Include native joins and cascading retirements in the + // drain condition, rather than using a FIFO marker. + worker_state.pending.fetch_sub(1, Ordering::Release); + } + })); + if outcome.is_err() { + fatal("route retirement worker panicked"); + } + }) + .unwrap_or_else(|_| fatal("could not start route retirement worker")); + // Native logging globals are constructed before these route owners. + // Drain queued destruction before those globals are destroyed. + // SAFETY: this C callback never unwinds and uses process-lifetime state. + // It does not close admission for cascading owner releases. + if unsafe { atexit(shutdown) } != 0 { + fatal("could not register route retirement exit drain"); + } + Retirement { sender, state } + }) +} + +pub(super) fn register_owner() { + worker().state.live.fetch_add(1, Ordering::AcqRel); +} + +pub(super) fn retire(owners: RouteOwners) { + let worker = worker(); + worker.state.pending.fetch_add(1, Ordering::AcqRel); + worker.state.pending_metric.inc(); + if let Err(undelivered) = worker.sender.send(Task { + owners, + queued_at: Instant::now(), + }) { + // A dead receiver must never drop these owners on the enqueueing RPC + // thread. Retain them until abort instead of recreating a self-join. + std::mem::forget(undelivered); + fatal("route retirement receiver unexpectedly closed"); + } +} + +fn fatal(message: &str) -> ! { + eprintln!("critical: {message}; next step: inspect RPC lifecycle worker failure"); + std::process::abort(); +} + +pub(super) fn pending() -> usize { + WORKER + .get() + .map_or(0, |worker| worker.state.pending.load(Ordering::Acquire)) +} + +pub(super) fn drain(timeout: std::time::Duration) -> bool { + let deadline = Instant::now() + timeout; + while pending() != 0 { + if Instant::now() >= deadline { + return false; + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } + true +} + +pub(super) fn quiescent() -> bool { + WORKER + .get() + .is_none_or(|worker| worker.state.live.load(Ordering::Acquire) == 0) +} + +pub(crate) extern "C" fn shutdown() { + // Explicit logging shutdown already requires quiescing callbacks and + // dropping instances first. Normal exit has also released Rust stack and + // runtime owners. Do not wait indefinitely for arbitrary live wrappers: + // process::exit intentionally leaves stack variables alive. Fail explicitly + // rather than run native static destructors under their active RPC workers. + if !drain(std::time::Duration::from_secs(30)) { + fatal("RPC route owner teardown did not drain before native shutdown"); + } + if !quiescent() { + fatal("live RPC route owners remain at native shutdown; quiesce and drop routes first"); + } +} diff --git a/lib/crowdb-rpc/ffi/tests/common/owned_route.rs b/lib/crowdb-rpc/ffi/tests/common/owned_route.rs new file mode 100644 index 000000000..fe7202ebc --- /dev/null +++ b/lib/crowdb-rpc/ffi/tests/common/owned_route.rs @@ -0,0 +1,184 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::ffi::c_void; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{mpsc, Arc}; +use std::thread::{self, ThreadId}; +use std::time::{Duration, Instant}; + +use arc_swap::ArcSwapOption; +use crowdb_protocol::fb::{ConnectionPingRequest, ConnectionPingRequestArgs}; +use crowdb_rpc_ffi::{sys, Buffer, OwnedClientRoute, RpcClient, RpcClientHandle, RpcServer}; +use flatbuffers::FlatBufferBuilder; + +#[derive(Clone, Copy)] +pub enum TestReply { + Success, + Close, + Timeout, +} + +struct TestClient { + rpc: Option, + destroyed: mpsc::Sender, +} + +impl RpcClientHandle for TestClient { + fn rpc_client_handle(&self) -> *mut c_void { + self.rpc.as_ref().unwrap().handle().cast() + } +} + +impl Drop for TestClient { + fn drop(&mut self) { + drop(self.rpc.take()); + let _ = self.destroyed.send(thread::current().id()); + } +} + +struct TestCompletion { + lease: OwnedClientRoute, + evicted: Arc, + completed: mpsc::Sender<(i32, ThreadId, bool)>, +} + +unsafe extern "C" fn complete( + _request: u64, + control: sys::crowdb_rpc_buffer_t, + data: sys::crowdb_rpc_buffer_t, + status: i32, + context: *mut c_void, +) { + // SAFETY: send transfers this unique box to exactly one completion. Both + // response handles also belong to this callback, including error outcomes. + let completion = unsafe { Box::from_raw(context.cast::()) }; + for buffer in [control, data] { + if !buffer.is_null() { + unsafe { sys::crowdb_rpc_buffer_release(buffer) }; + } + } + drop(completion.lease); + let _ = completion.completed.send(( + status, + thread::current().id(), + completion.evicted.load(Ordering::Acquire), + )); +} + +pub fn check_evicted_route(reply: TestReply) { + let rpc = RpcClient::new(); + rpc.set_completion_pool_size(8); + rpc.start_reaper(1_000_000_000, 10_000_000); + let remote = Arc::new(RpcServer::new(None)); + remote.listen("127.0.0.1", 0).unwrap(); + let release = Arc::new(AtomicBool::new(false)); + let (entered_tx, entered_rx) = mpsc::channel(); + remote.register_owned_handler(31011, { + let release = Arc::clone(&release); + move |request| { + let _ = entered_tx.send(()); + let deadline = Instant::now() + Duration::from_secs(3); + while !release.load(Ordering::Acquire) && Instant::now() < deadline { + thread::sleep(Duration::from_millis(1)); + } + match reply { + TestReply::Success => { + let _ = request.reply(control(), None, 31012); + } + TestReply::Close => request.close_connection(), + TestReply::Timeout => {} + } + } + }); + remote.start(); + let server = Arc::new(RpcServer::new(None)); + server.start(); + let connection = server.connect("127.0.0.1", remote.port()).unwrap(); + rpc.attach(&connection); + let (destroyed_tx, destroyed_rx) = mpsc::channel(); + let client = Arc::new(TestClient { + rpc: Some(rpc), + destroyed: destroyed_tx, + }); + let weak_client = Arc::downgrade(&client); + let weak_server = Arc::downgrade(&server); + let cache = ArcSwapOption::from(Some(Arc::new(OwnedClientRoute::new( + Arc::clone(&client), + Arc::clone(&server), + connection.clone(), + )))); + let (completed_tx, completed_rx) = mpsc::channel(); + let evicted = Arc::new(AtomicBool::new(false)); + let lease = cache.load_full().unwrap().as_ref().clone(); + let handles = lease.raw_handles(); + let completion = Box::into_raw(Box::new(TestCompletion { + lease, + evicted: Arc::clone(&evicted), + completed: completed_tx, + })); + // Model the native transport: its lease is the sole owner before submission. + // Setup cannot consume the request's timeout before cache eviction. + drop(cache.swap(None)); + drop(connection); + drop(client); + drop(server); + assert!(weak_client.upgrade().is_some()); + assert!(weak_server.upgrade().is_some()); + evicted.store(true, Ordering::Release); + // SAFETY: the unique completion lease retains all matching raw handles; + // success transfers the box to one callback, rejection leaves it to us. + let submitted = unsafe { + sys::crowdb_rpc_client_send( + handles.0.cast(), + handles.1.cast(), + handles.2.cast(), + 1, + control().into_raw(), + std::ptr::null_mut(), + 31011, + Some(complete), + completion.cast(), + ) + }; + if submitted != sys::CROWDB_RPC_OK { + // SAFETY: a rejected submission guarantees callback ownership was not + // transferred, so the caller must reclaim the original box. + drop(unsafe { Box::from_raw(completion) }); + } + assert_eq!(submitted, sys::CROWDB_RPC_OK); + entered_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + release.store(true, Ordering::Release); + let (status, callback_thread, after_eviction) = + completed_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + assert!(after_eviction, "completion ran before last-owner eviction"); + let expected = match reply { + TestReply::Success => sys::CROWDB_RPC_OK, + TestReply::Close => sys::CROWDB_RPC_ERR_CONN_CLOSED, + TestReply::Timeout => sys::CROWDB_RPC_ERR_TIMEOUT, + }; + assert_eq!(status, expected); + let destructor_thread = destroyed_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + assert_ne!(destructor_thread, callback_thread); + assert!(weak_client.upgrade().is_none()); + let deadline = Instant::now() + Duration::from_secs(3); + while weak_server.upgrade().is_some() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(1)); + } + assert!(weak_server.upgrade().is_none()); + assert!(OwnedClientRoute::drain_retired_for_tests(Duration::from_secs(3))); + remote.stop(); +} + +fn control() -> Buffer { + let mut builder = FlatBufferBuilder::new(); + let ping = ConnectionPingRequest::create( + &mut builder, + &ConnectionPingRequestArgs { + id: 1, + rpc_create_nano: 0, + }, + ); + builder.finish(ping, None); + Buffer::from_bytes(builder.finished_data()) +} diff --git a/lib/crowdb-rpc/ffi/tests/common/route_retirement.rs b/lib/crowdb-rpc/ffi/tests/common/route_retirement.rs new file mode 100644 index 000000000..962ad4301 --- /dev/null +++ b/lib/crowdb-rpc/ffi/tests/common/route_retirement.rs @@ -0,0 +1,79 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::ffi::c_void; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{mpsc, Arc}; +use std::time::{Duration, Instant}; + +use crowdb_rpc_ffi::{OwnedClientRoute, RpcClient, RpcClientHandle, RpcServer}; + +pub enum TestCleanup { + None, + ExitDelay, + Cascade(Option), + Block { + entered: mpsc::Sender<()>, + release: Arc, + }, +} + +pub struct TestReleaseGuard(pub Arc); + +impl Drop for TestReleaseGuard { + fn drop(&mut self) { + self.0.store(true, Ordering::Release); + } +} + +struct TestClient { + rpc: Option, + cleanup: TestCleanup, + released: mpsc::Sender<()>, +} + +impl RpcClientHandle for TestClient { + fn rpc_client_handle(&self) -> *mut c_void { + self.rpc.as_ref().unwrap().handle().cast() + } +} + +impl Drop for TestClient { + fn drop(&mut self) { + match &mut self.cleanup { + TestCleanup::None => {} + TestCleanup::ExitDelay => { + std::thread::sleep(Duration::from_millis(100)); + eprintln!("ROUTE_RETIREMENT_EXIT_DRAINED"); + } + TestCleanup::Cascade(child) => drop(child.take()), + TestCleanup::Block { entered, release } => { + let _ = entered.send(()); + let deadline = Instant::now() + Duration::from_secs(3); + while !release.load(Ordering::Acquire) && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(1)); + } + } + } + drop(self.rpc.take()); + let _ = self.released.send(()); + } +} + +pub fn route(cleanup: TestCleanup, released: mpsc::Sender<()>) -> OwnedClientRoute { + let server = Arc::new(RpcServer::new(None)); + server.listen("127.0.0.1", 0).unwrap(); + server.start(); + let connection = server.connect("127.0.0.1", server.port()).unwrap(); + let rpc = RpcClient::new(); + rpc.attach(&connection); + OwnedClientRoute::new( + Arc::new(TestClient { + rpc: Some(rpc), + cleanup, + released, + }), + server, + connection, + ) +} diff --git a/lib/crowdb-rpc/ffi/tests/owned_route_lifecycle_test.rs b/lib/crowdb-rpc/ffi/tests/owned_route_lifecycle_test.rs new file mode 100644 index 000000000..55af62e0c --- /dev/null +++ b/lib/crowdb-rpc/ffi/tests/owned_route_lifecycle_test.rs @@ -0,0 +1,22 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/owned_route.rs"] +mod owned_route; + +use owned_route::{check_evicted_route, TestReply}; + +#[test] +fn evicted_route_releases_last_owners_after_response_callback() { + check_evicted_route(TestReply::Success); +} + +#[test] +fn evicted_route_releases_last_owners_after_close_callback() { + check_evicted_route(TestReply::Close); +} + +#[test] +fn evicted_route_releases_last_owners_after_timeout_callback() { + check_evicted_route(TestReply::Timeout); +} diff --git a/lib/crowdb-rpc/ffi/tests/route_retirement_test.rs b/lib/crowdb-rpc/ffi/tests/route_retirement_test.rs new file mode 100644 index 000000000..dff49d2a7 --- /dev/null +++ b/lib/crowdb-rpc/ffi/tests/route_retirement_test.rs @@ -0,0 +1,111 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/route_retirement.rs"] +mod route_retirement; + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{mpsc, Arc}; +use std::time::Duration; + +use crowdb_rpc_ffi::OwnedClientRoute; +use route_retirement::{route, TestCleanup, TestReleaseGuard}; + +#[test] +fn normal_process_exit_drains_queued_owners_before_native_statics() { + const CHILD: &str = "CROWDB_ROUTE_EXIT_CHILD"; + if let Ok(mode) = std::env::var(CHILD) { + let (released, _receiver) = mpsc::channel(); + let route = route(TestCleanup::ExitDelay, released); + if mode == "live" { + std::mem::forget(route); + std::process::exit(0); + } + drop(route); + if mode == "logging" { + crowdb_rpc_ffi::shutdown_logging(); + } + return; + } + for mode in ["normal", "logging", "live"] { + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "normal_process_exit_drains_queued_owners_before_native_statics", + "--nocapture", + ]) + .env(CHILD, mode) + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + if mode == "live" { + assert!(!output.status.success(), "{stdout}\n{stderr}"); + assert!( + stderr.contains("live RPC route owners remain at native shutdown"), + "{stdout}\n{stderr}" + ); + continue; + } + assert!(output.status.success(), "{stdout}\n{stderr}"); + assert!( + stderr.contains("ROUTE_RETIREMENT_EXIT_DRAINED"), + "{stdout}\n{stderr}" + ); + assert!(!stderr.contains("LOG ERROR"), "{stdout}\n{stderr}"); + } +} + +#[test] +fn request_clones_keep_one_owner_bundle_until_the_last_release() { + let (released_tx, released_rx) = mpsc::channel(); + let route = route(TestCleanup::None, released_tx); + let clones = vec![route.clone(); 256]; + let handles = route.raw_handles(); + assert!(clones.iter().all(|route| route.raw_handles() == handles)); + drop(clones); + assert!(released_rx.try_recv().is_err()); + assert!(route.is_open()); + drop(route); + released_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + assert!(OwnedClientRoute::drain_retired_for_tests(Duration::from_secs(3))); + assert!(released_rx.try_recv().is_err()); +} + +#[test] +fn cascading_retirement_drains_and_a_stalled_destructor_keeps_backlog_visible() { + let (released_tx, released_rx) = mpsc::channel(); + let (entered_tx, entered_rx) = mpsc::channel(); + let release = Arc::new(AtomicBool::new(false)); + let guard = TestReleaseGuard(Arc::clone(&release)); + let child = route( + TestCleanup::Block { + entered: entered_tx, + release: Arc::clone(&release), + }, + released_tx.clone(), + ); + let parent = route(TestCleanup::Cascade(Some(child)), released_tx.clone()); + drop(parent); + entered_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + assert!(OwnedClientRoute::pending_retired_for_tests() >= 1); + assert!(!OwnedClientRoute::drain_retired_for_tests(Duration::from_millis( + 10 + ))); + let tail = route(TestCleanup::None, released_tx); + let (queued_tx, queued_rx) = mpsc::channel(); + let producer = std::thread::spawn(move || { + drop(tail); + queued_tx.send(()).unwrap(); + }); + queued_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + assert!(OwnedClientRoute::pending_retired_for_tests() >= 2); + release.store(true, Ordering::Release); + drop(guard); + producer.join().unwrap(); + assert!(OwnedClientRoute::drain_retired_for_tests(Duration::from_secs(3))); + for _ in 0..3 { + released_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + } + assert!(released_rx.try_recv().is_err()); +} diff --git a/lib/crowdb-rpc/src/c_api.cpp b/lib/crowdb-rpc/src/c_api.cpp index f62655bf2..4823cee9a 100644 --- a/lib/crowdb-rpc/src/c_api.cpp +++ b/lib/crowdb-rpc/src/c_api.cpp @@ -443,6 +443,12 @@ crowdb_rpc_status crowdb_rpc_client_send(crowdb_rpc_client_t client, crowdb_rpc_ return CROWDB_RPC_ERR_INVALID_ARG; } + // Completion can synchronously destroy all caller-side wrappers. Keep + // their native owners through submission and never read wrappers after it. + const auto transport = server->server->shared_transport(); + const auto caller = client->client; + const auto connection = conn->conn; + crowdb::rpc::Buffer *ctrl_buf = control->buf; crowdb::rpc::Buffer *data_buf = (data != nullptr) ? data->buf : nullptr; @@ -454,8 +460,8 @@ crowdb_rpc_status crowdb_rpc_client_send(crowdb_rpc_client_t client, crowdb_rpc_ data_buf->ref_clone(); } - bool ok = client->client->send(server->server->transport(), conn->conn.get(), request_id, ctrl_buf, data_buf, - msg_type, on_complete, user_data); + bool ok = caller->send(transport.get(), connection.get(), request_id, ctrl_buf, data_buf, msg_type, on_complete, + user_data); // Release the caller's wrapper handles (decrements the ref bumped // above and frees the crowdb_rpc_buffer_s struct). @@ -464,7 +470,7 @@ crowdb_rpc_status crowdb_rpc_client_send(crowdb_rpc_client_t client, crowdb_rpc_ crowdb_rpc_buffer_release(data); } - return submit_status(ok, conn->conn.get()); + return submit_status(ok, connection.get()); } catch (...) { return CROWDB_RPC_ERR_CONN_ERROR; @@ -487,6 +493,10 @@ crowdb_rpc_status crowdb_rpc_client_send_chain(crowdb_rpc_client_t client, crowd return CROWDB_RPC_ERR_INVALID_ARG; } + const auto transport = server->server->shared_transport(); + const auto caller = client->client; + const auto connection = conn->conn; + std::array buffers{}; uint64_t total = 0; for (uint8_t index = 0; index < data_view_count; ++index) { @@ -504,14 +514,14 @@ crowdb_rpc_status crowdb_rpc_client_send_chain(crowdb_rpc_client_t client, crowd for (uint8_t index = 0; index < data_view_count; ++index) { buffers[index]->ref_clone(); } - bool ok = client->client->send_chain(server->server->transport(), conn->conn.get(), request_id, control->buf, - buffers.data(), data_view_count, msg_type, on_complete, user_data); + bool ok = caller->send_chain(transport.get(), connection.get(), request_id, control->buf, buffers.data(), + data_view_count, msg_type, on_complete, user_data); crowdb_rpc_buffer_release(control); for (uint8_t index = 0; index < data_view_count; ++index) { crowdb_rpc_buffer_release(data_views[index]); } - return submit_status(ok, conn->conn.get()); + return submit_status(ok, connection.get()); } catch (...) { return CROWDB_RPC_ERR_CONN_ERROR; @@ -528,6 +538,10 @@ crowdb_rpc_status crowdb_rpc_client_send_slab(crowdb_rpc_client_t client, crowdb return CROWDB_RPC_ERR_INVALID_ARG; } + const auto transport = server->server->shared_transport(); + const auto caller = client->client; + const auto connection = conn->conn; + crowdb::rpc::Buffer *ctrl_buf = control->buf; crowdb::rpc::Buffer *data_buf = (data != nullptr) ? data->buf : nullptr; if (ctrl_buf != nullptr) { @@ -537,13 +551,13 @@ crowdb_rpc_status crowdb_rpc_client_send_slab(crowdb_rpc_client_t client, crowdb data_buf->ref_clone(); } - bool ok = client->client->send_slab_only(server->server->transport(), conn->conn.get(), request_id, ctrl_buf, - data_buf, msg_type, on_complete, user_data); + bool ok = caller->send_slab_only(transport.get(), connection.get(), request_id, ctrl_buf, data_buf, msg_type, + on_complete, user_data); crowdb_rpc_buffer_release(control); if (data != nullptr) { crowdb_rpc_buffer_release(data); } - return submit_status(ok, conn->conn.get()); + return submit_status(ok, connection.get()); } catch (...) { return CROWDB_RPC_ERR_CONN_ERROR; diff --git a/lib/crowdb-rpc/tests/c_api_submit_lifetime_test.cpp b/lib/crowdb-rpc/tests/c_api_submit_lifetime_test.cpp new file mode 100644 index 000000000..7c8dcfc17 --- /dev/null +++ b/lib/crowdb-rpc/tests/c_api_submit_lifetime_test.cpp @@ -0,0 +1,124 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/c_api_internal.h" +#include "crowdb-rpc/transport/socket_transport.h" + +#include +#include +#include + +#include +#include + +namespace +{ +using namespace crowdb::rpc; + +enum class TestSubmit : uint8_t { Call, Chain, Slab }; + +struct TestCompletion +{ + crowdb_rpc_client_t client = nullptr; + crowdb_rpc_server_t server = nullptr; + crowdb_rpc_conn_t connection = nullptr; + std::shared_ptr transport; + uint32_t calls = 0; + bool returned = false; + + ~TestCompletion() + { + crowdb_rpc_client_destroy(client); + crowdb_rpc_server_destroy(server); + crowdb_rpc_conn_destroy(connection); + } +}; + +void complete(uint64_t /*request*/, crowdb_rpc_buffer_t control, crowdb_rpc_buffer_t data, crowdb_rpc_status status, + void *context) +{ + auto &state = *static_cast(context); + ++state.calls; + EXPECT_FALSE(state.returned); + EXPECT_EQ(status, CROWDB_RPC_ERR_CONN_CLOSED); + EXPECT_EQ(control, nullptr); + EXPECT_EQ(data, nullptr); + crowdb_rpc_client_destroy(state.client); + state.client = nullptr; + crowdb_rpc_server_destroy(state.server); + state.server = nullptr; + crowdb_rpc_conn_destroy(state.connection); + state.connection = nullptr; + // The fixture retains one native owner. The active ABI call must retain + // another, even after a synchronous callback destroys every C wrapper. + EXPECT_GE(state.transport.use_count(), 2); +} + +void check_submit(TestSubmit method, bool pooled) +{ + SystemBufferPool pool; + TestCompletion state; + state.server = new crowdb_rpc_server_s{new RpcServer(&pool)}; + state.transport = state.server->server->shared_transport(); + std::array sockets{}; + ASSERT_EQ(::socketpair(AF_UNIX, SOCK_STREAM, 0, sockets.data()), 0); + auto native_connection = state.transport->create_connection(sockets[0], "synchronous-close"); + state.connection = new crowdb_rpc_conn_s{native_connection}; + state.client = crowdb_rpc_client_create(); + ASSERT_NE(state.client, nullptr); + const auto native_client = state.client->client; + if (pooled) { + crowdb_rpc_client_set_completion_pool_size(state.client, 8); + } + crowdb_rpc_client_attach(state.client, state.connection); + const int original_write_fd = native_connection->write_fd; + // This descriptor is outside the process descriptor limit. The real + // direct-write path gets EBADF and synchronously completes on close. + native_connection->write_fd = std::numeric_limits::max(); + constexpr std::array payload{1, 2, 3, 4, 5, 6, 7}; + auto *const control = crowdb_rpc_buffer_create(payload.data(), payload.size()); + crowdb_rpc_status status = CROWDB_RPC_ERR_INVALID_ARG; + if (method == TestSubmit::Chain) { + const std::array data{crowdb_rpc_buffer_create(payload.data(), 3), + crowdb_rpc_buffer_create(payload.data() + 3, 3)}; + status = crowdb_rpc_client_send_chain(state.client, state.server, state.connection, 1, control, data.data(), + data.size(), 42, complete, &state); + } + else if (method == TestSubmit::Slab) { + status = crowdb_rpc_client_send_slab(state.client, state.server, state.connection, 1, control, nullptr, 42, + complete, &state); + } + else { + status = crowdb_rpc_client_send(state.client, state.server, state.connection, 1, control, nullptr, 42, complete, + &state); + } + state.returned = true; + native_connection->write_fd = original_write_fd; + EXPECT_EQ(status, CROWDB_RPC_OK); + EXPECT_EQ(state.calls, 1U); + EXPECT_EQ(native_client->pending_count(), 0U); + EXPECT_FALSE(native_connection->is_open()); + state.transport->stop(); + ::close(sockets[1]); +} +} // namespace + +TEST(CApiSubmitLifetimeTest, MapCompletionMayDestroyWrappersBeforeSendReturns) +{ + check_submit(TestSubmit::Call, false); +} + +TEST(CApiSubmitLifetimeTest, SlabCompletionMayDestroyWrappersBeforeSendReturns) +{ + check_submit(TestSubmit::Call, true); +} + +TEST(CApiSubmitLifetimeTest, ChainCompletionMayDestroyWrappersBeforeSendReturns) +{ + check_submit(TestSubmit::Chain, true); +} + +TEST(CApiSubmitLifetimeTest, SlabOnlyCompletionMayDestroyWrappersBeforeSendReturns) +{ + check_submit(TestSubmit::Slab, true); +} diff --git a/lib/crowdb-tree/CMakeLists.txt b/lib/crowdb-tree/CMakeLists.txt index d8b2ebe2f..26c54fce6 100644 --- a/lib/crowdb-tree/CMakeLists.txt +++ b/lib/crowdb-tree/CMakeLists.txt @@ -100,7 +100,9 @@ include(GoogleTest) file(GLOB_RECURSE CROWDB_TREE_TEST_SRC CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/tests/*.cpp) if(NOT (LIBURING_INCLUDE_DIR AND LIBURING_LIB)) - list(REMOVE_ITEM CROWDB_TREE_TEST_SRC ${CMAKE_CURRENT_SOURCE_DIR}/tests/unit/reactor_test.cpp) + list(REMOVE_ITEM CROWDB_TREE_TEST_SRC + ${CMAKE_CURRENT_SOURCE_DIR}/tests/unit/reactor_test.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/unit/block_async_page_store_test.cpp) endif() list(APPEND CROWDB_TREE_TEST_SRC ${CMAKE_CURRENT_SOURCE_DIR}/../crowdb-common/cpp/tests/test_main.cpp) add_executable(crowdb_tree_tests ${CROWDB_TREE_TEST_SRC}) diff --git a/lib/crowdb-tree/ffi/src/tree.rs b/lib/crowdb-tree/ffi/src/tree.rs index 54e26573d..8f36a39d5 100644 --- a/lib/crowdb-tree/ffi/src/tree.rs +++ b/lib/crowdb-tree/ffi/src/tree.rs @@ -559,6 +559,11 @@ pub fn ct_flush_logging() { /// buffered log messages are written to disk. Safe to call when logging /// was never initialized (no-op in that case). pub fn ct_shutdown_logging() { + #[cfg(feature = "chunk-rpc")] + assert!( + crowdb_rpc_ffi::OwnedClientRoute::drain_retired(std::time::Duration::from_secs(30)), + "RPC route owner teardown did not drain before native logging shutdown" + ); unsafe { sys::ct_shutdown_logging() }; } diff --git a/lib/crowdb-tree/include/crowdb-tree/btree/tree.h b/lib/crowdb-tree/include/crowdb-tree/btree/tree.h index bc95ff3f1..267feb80d 100644 --- a/lib/crowdb-tree/include/crowdb-tree/btree/tree.h +++ b/lib/crowdb-tree/include/crowdb-tree/btree/tree.h @@ -717,8 +717,10 @@ class Crowdbtree void commit_prepared_snapshot(const PreparedSnapshot &prepared); void finalize_prepared_snapshot(PreparedSnapshot &prepared); - void acquire_snapshot_slot(); - void release_snapshot_slot(); + void acquire_snapshot_slot(); + void release_snapshot_slot(); + Status retry_snapshot_anchor(); + void snapshot_prepare_async(std::function on_done); void snapshot_write_next_async(std::shared_ptr prepared, size_t idx, std::function on_done); @@ -833,6 +835,9 @@ class Crowdbtree // snapshot_async's async write phase, where write_mutex_ itself can't be // held (see acquire_snapshot_slot's doc comment and snapshot_async's). std::atomic snapshot_inflight_{false}; + // Protected by snapshot_inflight_: retry an uncertain anchor barrier before + // preparing or writing another image over the backend's staged snapshot. + bool snapshot_anchor_pending_ = false; // Block indices that were empty in the previous snapshot (two-generation // rule for block deletion). A block is only deleted after it's empty in diff --git a/lib/crowdb-tree/include/crowdb-tree/maptable/buffer_pool.h b/lib/crowdb-tree/include/crowdb-tree/maptable/buffer_pool.h index dd997f8e5..47fbd979a 100644 --- a/lib/crowdb-tree/include/crowdb-tree/maptable/buffer_pool.h +++ b/lib/crowdb-tree/include/crowdb-tree/maptable/buffer_pool.h @@ -23,6 +23,7 @@ #include #include +#include #include #include @@ -178,7 +179,7 @@ class BufferPool [[nodiscard]] uint8_t *frame_bytes(uint32_t idx) { - return arena_.data() + (static_cast(idx) * page_bytes_); + return arena_.get() + (static_cast(idx) * page_bytes_); } void unpin(uint32_t idx); @@ -193,14 +194,14 @@ class BufferPool [[nodiscard]] int64_t acquire_victim(); Status write_back_io(uint32_t idx, PageAddr addr); - mutable std::mutex mu_; - std::condition_variable state_cv_; - std::vector arena_; - std::vector frames_; - uint32_t page_bytes_; - uint32_t num_frames_; - PageStore *store_; - uint32_t clock_hand_ = 0; + mutable std::mutex mu_; + std::condition_variable state_cv_; + std::unique_ptr arena_; + std::vector frames_; + uint32_t page_bytes_; + uint32_t num_frames_; + PageStore *store_; + uint32_t clock_hand_ = 0; std::vector ht_key_; // page_id or kInvalidPageId std::vector ht_val_; // frame index diff --git a/lib/crowdb-tree/include/crowdb-tree/snapshot/prepared.h b/lib/crowdb-tree/include/crowdb-tree/snapshot/prepared.h index f5fe4b6d8..64d62ef00 100644 --- a/lib/crowdb-tree/include/crowdb-tree/snapshot/prepared.h +++ b/lib/crowdb-tree/include/crowdb-tree/snapshot/prepared.h @@ -40,7 +40,7 @@ struct PreparedPageWrite uint64_t prior_addr = kNoAddr; uint64_t addr = 0; uint32_t logical_len = 0; // unpadded; mirrors PageBase::durable_plen - std::vector blob; // already IU-padded + std::vector blob; // padded to IU at submission; released after completion }; // A dirty MappingSegment's fresh image write, plus enough identity to @@ -80,10 +80,11 @@ struct PrefetchedPage std::vector blob; // IU-padded page content read from the store }; -// Output of prepare_snapshot_locked(): every byte this snapshot generation -// needs written, computed synchronously under write_mutex_ (the segment +// Output of prepare_snapshot_locked(): encoded page payloads and metadata +// bytes computed synchronously under write_mutex_ (the segment // scan + delta-fold + page/segment-image/directory encode is CPU/memory-only -// -- see the "Lock scope" note on #11). The caller writes +// without device padding). Page payloads gain device padding +// only at submission. The caller writes // `page_writes` and `segment_writes` (any order/concurrency) then // `directory_write`, then a durability barrier, then `anchor_write` -- // writing the anchor before that barrier would violate the crash-safety diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_manifest_build.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_manifest_build.cpp index 8252ff5da..f7f68b1d5 100644 --- a/lib/crowdb-tree/src/backend/chunk/chunk_manifest_build.cpp +++ b/lib/crowdb-tree/src/backend/chunk/chunk_manifest_build.cpp @@ -128,10 +128,14 @@ Status ChunkPageStore::build_manifest(uint64_t expected_generation, std::shared_ pack.ref.length = static_cast(length); const size_t physical_length = chunk_framed_length(length); std::vector framed; + std::vector payload; + Status staging_status = read_staged_range(offset, length, &payload, cancellation); + if (!staging_status.ok()) { + return staging_status; + } if (physical_length == 0 || crowdb::protocol::encode_frames(chunk_page_frame_magic(purpose), - {.high = active_chunk_id_.high, .low = active_chunk_id_.low}, - std::span(staged_.data() + offset, length), + {.high = active_chunk_id_.high, .low = active_chunk_id_.low}, payload, monotonic_millis(), &framed) != crowdb::protocol::FrameError::Ok) { return Status::resource_exhausted("chunk page frame encoding failed"); } diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_pack_pipeline.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_pack_pipeline.cpp index 483c3135c..d07b4925d 100644 --- a/lib/crowdb-tree/src/backend/chunk/chunk_pack_pipeline.cpp +++ b/lib/crowdb-tree/src/backend/chunk/chunk_pack_pipeline.cpp @@ -392,6 +392,7 @@ class ChunkPackPipelineImpl final : public ChunkPackPipeline, public std::enable std::memory_order_release); store->staged_initialized_ = false; store->staged_.clear(); + store->staged_base_.reset(); store->dirty_ranges_.clear(); store->inherited_manifest_.reset(); store->inherited_catalog_.reset(); @@ -405,11 +406,18 @@ class ChunkPackPipelineImpl final : public ChunkPackPipeline, public std::enable { auto &job = *writes[index]; orphan_pack_bytes.fetch_add(job.pack.ref.length, std::memory_order_relaxed); - const auto frame_error = crowdb::protocol::encode_frames( - chunk_page_frame_magic(chunk_purpose(job.pack.ref.chunk_id)), - {.high = job.pack.ref.chunk_id.high, .low = job.pack.ref.chunk_id.low}, - std::span(store->staged_.data() + job.source_offset, job.pack.ref.length), - monotonic_millis(), &job.framed); + std::vector payload; + Status staging_status = + store->read_staged_range(job.source_offset, job.pack.ref.length, &payload, cancellation); + if (!staging_status.ok()) { + stop_requested.store(true, std::memory_order_release); + completion.complete(std::move(staging_status)); + return; + } + const auto frame_error = + crowdb::protocol::encode_frames(chunk_page_frame_magic(chunk_purpose(job.pack.ref.chunk_id)), + {.high = job.pack.ref.chunk_id.high, .low = job.pack.ref.chunk_id.low}, + payload, monotonic_millis(), &job.framed); if (frame_error != crowdb::protocol::FrameError::Ok || job.framed.size() != job.physical_length) { stop_requested.store(true, std::memory_order_release); completion.complete(Status::corruption("chunk page frame encoding failed")); diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_page_purpose.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_page_purpose.cpp index 59a710bfb..5ed8d0fac 100644 --- a/lib/crowdb-tree/src/backend/chunk/chunk_page_purpose.cpp +++ b/lib/crowdb-tree/src/backend/chunk/chunk_page_purpose.cpp @@ -32,12 +32,21 @@ Status ChunkPageStore::write_typed_at(PagePurpose purpose, uint64_t off, const u return Status::resource_exhausted("chunk page write exceeds address space"); } if (!staged_initialized_) { - Status status = materialize_active(&staged_); - if (!status.ok()) { - return status; + staged_base_ = reuse_base_manifest(); + if (staged_base_ != nullptr) { + if (staged_base_->owner_epoch > config_.owner_epoch) { + return Status::corruption("chunk manifest checksum or epoch is invalid"); + } + const RootCatalog &base_catalog = + staged_base_ == inherited_manifest_ && inherited_catalog_ != nullptr ? *inherited_catalog_ : *catalog_; + Status status = validate_manifest(*staged_base_, base_catalog); + if (!status.ok()) { + return status; + } } + staged_.reset(staged_base_ == nullptr ? 0 : staged_base_->logical_size); staged_purposes_.clear(); - auto base = reuse_base_manifest(); + auto base = staged_base_; if (base != nullptr) { for (const auto &pack : base->packs) { staged_purposes_.assign(pack.logical_offset, pack.ref.length, chunk_purpose(pack.ref.chunk_id)); @@ -45,12 +54,13 @@ Status ChunkPageStore::write_typed_at(PagePurpose purpose, uint64_t off, const u } staged_initialized_ = true; } - const size_t end = static_cast(off) + len; - if (end > staged_.size()) { - staged_.resize(end, 0); + const size_t end = static_cast(off) + len; + Status status = prepare_staged_write(off, len); + if (!status.ok()) { + return status; } + staged_.write(off, std::span(buf, len)); if (len != 0) { - std::memcpy(staged_.data() + off, buf, len); dirty_ranges_.emplace_back(off, len); staged_purposes_.assign(off, len, purpose); } diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_page_staging.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_page_staging.cpp new file mode 100644 index 000000000..fbb64cf58 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_page_staging.cpp @@ -0,0 +1,86 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "chunk_page_store.h" + +#include +#include + +namespace crowdb::tree::detail +{ + +Status ChunkPageStore::prepare_staged_write(uint64_t offset, size_t length) +{ + if (staged_base_ == nullptr || length == 0) { + return Status::Ok(); + } + const uint64_t end = offset + length; + for (const auto &pack : staged_base_->packs) { + const uint64_t pack_end = pack.logical_offset + pack.ref.length; + if (pack_end <= offset || pack.logical_offset >= end || (offset <= pack.logical_offset && end >= pack_end) || + staged_.covers(pack.logical_offset, pack.ref.length)) { + continue; + } + std::shared_ptr> persisted; + Status status = read_pack(pack.ref, &persisted, {}); + if (!status.ok()) { + return status; + } + std::vector captured(*persisted); + staged_.overlay(pack.logical_offset, captured); + staged_.write(pack.logical_offset, captured); + } + return Status::Ok(); +} + +Status ChunkPageStore::read_staged_range(uint64_t offset, size_t length, std::vector *out, + ChunkCancellation cancellation) const +{ + if (offset > staged_.size() || length > staged_.size() - offset) { + return Status::corruption("staged chunk range exceeds logical image"); + } + out->assign(length, 0); + if (staged_base_ != nullptr) { + const uint64_t end = offset + length; + for (const auto &pack : staged_base_->packs) { + const uint64_t begin = std::max(offset, pack.logical_offset); + const uint64_t stop = std::min(end, pack.logical_offset + pack.ref.length); + if (begin >= stop || staged_.covers(begin, stop - begin)) { + continue; + } + if (cancellation.cancelled()) { + return Status::unavailable("chunk staging read cancelled"); + } + std::shared_ptr> payload; + Status status = read_pack(pack.ref, &payload, cancellation); + if (!status.ok()) { + return status; + } + std::memcpy(out->data() + begin - offset, payload->data() + begin - pack.logical_offset, stop - begin); + } + } + staged_.overlay(offset, *out); + return cancellation.cancelled() ? Status::unavailable("chunk staging read cancelled") : Status::Ok(); +} + +const ChunkPagePack *ChunkPageStore::find_reusable_pack(const ChunkManifest &base, uint64_t logical_offset, + uint32_t length, ChunkCancellation cancellation) const +{ + const ChunkPagePack *found = find_pack_at(base, logical_offset, length); + if (found == nullptr) { + return nullptr; + } + std::shared_ptr> persisted; + Status status = read_pack(found->ref, &persisted, cancellation); + if (!status.ok() || persisted->size() != length) { + return nullptr; + } + if (!range_was_written(logical_offset, length)) { + return found; + } + std::vector overlaid(*persisted); + staged_.overlay(logical_offset, overlaid); + return overlaid == *persisted ? found : nullptr; +} + +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_page_staging.h b/lib/crowdb-tree/src/backend/chunk/chunk_page_staging.h new file mode 100644 index 000000000..1b8dcb02d --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_page_staging.h @@ -0,0 +1,151 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace crowdb::tree::detail +{ + +// Single-writer overlay. Fragments never cross a 64-KiB boundary, bounding +// temporary copies when a later write replaces part of an earlier fragment. +// Trailing zero bytes retain logical coverage without retaining padding buffers. +class ChunkPageStaging +{ + public: + void reset(uint64_t size) + { + fragments_.clear(); + size_ = size; + } + + void clear() + { + reset(0); + } + + [[nodiscard]] size_t size() const + { + return static_cast(size_); + } + + void write(uint64_t offset, std::span bytes) + { + size_ = std::max(size_, offset + bytes.size()); + while (!bytes.empty()) { + const auto length = std::min(bytes.size(), kFragmentBytes - (offset % kFragmentBytes)); + assign(offset, bytes.first(length)); + offset += length; + bytes = bytes.subspan(length); + } + } + + void overlay(uint64_t offset, std::span bytes) const + { + auto next = fragments_.upper_bound(offset); + if (next != fragments_.begin()) { + --next; + } + const uint64_t end = offset + bytes.size(); + for (; next != fragments_.end() && next->first < end; ++next) { + const uint64_t begin = std::max(offset, next->first); + const uint64_t stop = std::min(end, next->first + next->second.length); + if (begin < stop) { + auto target = bytes.subspan(begin - offset, stop - begin); + std::fill(target.begin(), target.end(), 0); + const uint64_t copied_end = std::min(stop, next->first + next->second.bytes.size()); + if (begin < copied_end) { + std::memcpy(target.data(), next->second.bytes.data() + begin - next->first, copied_end - begin); + } + } + } + } + + [[nodiscard]] bool covers(uint64_t offset, uint64_t length) const + { + const uint64_t end = offset + length; + auto next = fragments_.upper_bound(offset); + if (next != fragments_.begin()) { + --next; + } + for (; next != fragments_.end() && offset < end; ++next) { + if (next->first > offset) { + return false; + } + offset = std::max(offset, next->first + next->second.length); + } + return offset >= end; + } + + private: + struct Fragment + { + uint64_t length = 0; + std::vector bytes; + + static Fragment capture(std::span input) + { + const auto last = std::find_if(input.rbegin(), input.rend(), [](uint8_t byte) { return byte != 0; }); + const auto kept = input.size() - static_cast(std::distance(input.rbegin(), last)); + const auto retained = input.first(kept); + return {.length = input.size(), .bytes = std::vector(retained.begin(), retained.end())}; + } + + [[nodiscard]] Fragment slice(uint64_t offset, uint64_t size) const + { + Fragment result; + if (offset < bytes.size()) { + result = capture(std::span(bytes).subspan(offset, std::min(size, bytes.size() - offset))); + } + result.length = size; + return result; + } + }; + + void assign(uint64_t offset, std::span bytes) + { + const uint64_t end = offset + bytes.size(); + auto next = fragments_.upper_bound(offset); + if (next != fragments_.begin()) { + const auto previous = std::prev(next); + if (previous->first + previous->second.length > offset) { + next = previous; + } + } + Fragment prefix; + Fragment suffix; + uint64_t prefix_offset = offset; + while (next != fragments_.end() && next->first < end) { + if (next->first < offset) { + prefix_offset = next->first; + prefix = next->second.slice(0, offset - next->first); + } + if (next->first + next->second.length > end) { + suffix = next->second.slice(end - next->first, next->first + next->second.length - end); + } + next = fragments_.erase(next); + } + if (prefix.length > 0) { + fragments_.emplace(prefix_offset, std::move(prefix)); + } + if (suffix.length > 0) { + fragments_.emplace(end, std::move(suffix)); + } + fragments_.emplace(offset, Fragment::capture(bytes)); + } + + static constexpr uint64_t kFragmentBytes = 64U * 1024U; + uint64_t size_ = 0; + std::map fragments_; +}; + +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_page_store.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_page_store.cpp index 60292e5e5..a3eecd15b 100644 --- a/lib/crowdb-tree/src/backend/chunk/chunk_page_store.cpp +++ b/lib/crowdb-tree/src/backend/chunk/chunk_page_store.cpp @@ -850,6 +850,9 @@ bool ChunkPageStore::inherited_snapshot_matches(const PageStore &source_store) c std::shared_ptr ChunkPageStore::reuse_base_manifest() const { + if (staged_initialized_) { + return staged_base_; + } auto bootstrap = bootstrap_layout_.load(std::memory_order_acquire); if (bootstrap != nullptr) { return bootstrap; @@ -862,22 +865,6 @@ std::shared_ptr ChunkPageStore::reuse_base_manifest() const return current == nullptr ? inherited_manifest_ : current; } -const ChunkPagePack *ChunkPageStore::find_reusable_pack(const ChunkManifest &base, uint64_t logical_offset, - uint32_t length, ChunkCancellation cancellation) const -{ - const ChunkPagePack *found = find_pack_at(base, logical_offset, length); - if (found == nullptr) { - return nullptr; - } - std::shared_ptr> persisted; - Status status = read_pack(found->ref, &persisted, cancellation); - if (!status.ok() || persisted->size() != length || - std::memcmp(persisted->data(), staged_.data() + logical_offset, length) != 0) { - return nullptr; - } - return found; -} - const ChunkPagePack *ChunkPageStore::find_pack_at(const ChunkManifest &base, uint64_t logical_offset, uint32_t length) { auto found = @@ -939,38 +926,6 @@ Status ChunkPageStore::validate_manifest(const ChunkManifest &manifest, const Ro return Status::Ok(); } -Status ChunkPageStore::materialize_active(std::vector *out) const -{ - auto manifest = reuse_base_manifest(); - if (manifest == nullptr) { - out->clear(); - return Status::Ok(); - } - if (manifest->owner_epoch > config_.owner_epoch) { - return Status::corruption("chunk manifest checksum or epoch is invalid"); - } - const RootCatalog &manifest_catalog = - manifest == inherited_manifest_ && inherited_catalog_ != nullptr ? *inherited_catalog_ : *catalog_; - Status manifest_status = validate_manifest(*manifest, manifest_catalog); - if (!manifest_status.ok()) { - return manifest_status; - } - out->assign(manifest->logical_size, 0); - for (const ChunkPagePack &pack : manifest->packs) { - const ChunkPageRef &ref = pack.ref; - if (pack.logical_offset > out->size() || ref.length > out->size() - pack.logical_offset) { - return Status::corruption("chunk page pack checksum or bounds are invalid"); - } - std::shared_ptr> payload; - Status read_status = read_pack(ref, &payload, {}); - if (!read_status.ok()) { - return read_status; - } - std::memcpy(out->data() + pack.logical_offset, payload->data(), payload->size()); - } - return Status::Ok(); -} - Status ChunkPageStore::read_at(uint64_t off, uint8_t *buf, size_t len) const { return read_at_cancellable(off, buf, len, {}); @@ -1234,6 +1189,7 @@ Status ChunkPageStore::sync_cancellable(ChunkCancellation cancellation) layout_valid_until_ms_.store(monotonic_millis() + config_.layout_validity_ms, std::memory_order_release); staged_initialized_ = false; staged_.clear(); + staged_base_.reset(); dirty_ranges_.clear(); inherited_manifest_.reset(); inherited_catalog_.reset(); diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_page_store.h b/lib/crowdb-tree/src/backend/chunk/chunk_page_store.h index 8957f4b34..762a94bf5 100644 --- a/lib/crowdb-tree/src/backend/chunk/chunk_page_store.h +++ b/lib/crowdb-tree/src/backend/chunk/chunk_page_store.h @@ -6,6 +6,7 @@ #pragma once #include "chunk_cancellation.h" +#include "chunk_page_staging.h" #include "chunk_purpose.h" #include "chunk_transport.h" #include "crowdb-tree/backend/async_page_store.h" @@ -365,7 +366,9 @@ class ChunkPageStore final : public PageStore, public AsyncPageStore size_t staged_pack_length(uint64_t offset, const ChunkManifest *base) const; static bool range_has_pack(const ChunkManifest &base, uint64_t offset, size_t length); uint64_t chunk_allocation_bytes() const; - Status materialize_active(std::vector *out) const; + Status read_staged_range(uint64_t offset, size_t length, std::vector *out, + ChunkCancellation cancellation = {}) const; + Status prepare_staged_write(uint64_t offset, size_t length); Status refresh_active_chunk(); Status build_manifest(uint64_t expected_generation, std::shared_ptr *out, uint64_t *new_pack_bytes, ChunkCancellation cancellation = {}); @@ -393,7 +396,8 @@ class ChunkPageStore final : public PageStore, public AsyncPageStore Config config_; std::shared_ptr catalog_; std::shared_ptr transport_; - std::vector staged_; + ChunkPageStaging staged_; + std::shared_ptr staged_base_; PagePurposeRanges staged_purposes_; std::vector> dirty_ranges_; std::vector> materialization_live_extents_; diff --git a/lib/crowdb-tree/src/backend/local/block_async_page_store.cpp b/lib/crowdb-tree/src/backend/local/block_async_page_store.cpp index 167e63faf..502bd7de5 100644 --- a/lib/crowdb-tree/src/backend/local/block_async_page_store.cpp +++ b/lib/crowdb-tree/src/backend/local/block_async_page_store.cpp @@ -10,9 +10,12 @@ #include #include +#include #include #include #include +#include +#include #include #include @@ -23,22 +26,51 @@ namespace { // Maps a DiskIOUring callback's raw CQE `res` (>=0 bytes transferred, <0 // -errno) to a Status, treating a short read/write (res >= 0 but < the -// requested `len`) as an io_error too -- a single-shot op per -// submit_read/write call, unlike the synchronous stores' internal -// retry-until-full-length loop; a later phase can add that retry if real -// usage needs it. +// requested `len`) as an io_error too. Each extent is submitted once; +// crossing a block boundary does not retry a short read in the first file. Status result_to_status(int res, size_t len, const char *op) { if (res < 0) { return Status::io_error(std::string(op) + ": " + std::strerror(-res)); } - if (static_cast(res) < len) { + if (std::cmp_less(res, len)) { return Status::io_error(std::string("short ") + op + " (" + std::to_string(res) + " of " + std::to_string(len) + " bytes)"); } return Status::Ok(); } +struct ExtentRead +{ + int fd; + void *buffer; + size_t length; + uint64_t offset; +}; + +struct ExtentReads : std::enable_shared_from_this +{ + ::crowdb::common::DiskIOUring *uring = nullptr; + std::vector parts; + AsyncCompletion completion; + + void submit(size_t index) + { + const auto &part = parts[index]; + uring->submit_read(part.fd, part.buffer, part.length, static_cast(part.offset), + [state = shared_from_this(), index, length = part.length](int result) { + Status status = result_to_status(result, length, "read"); + if (!status.ok() || index + 1 == state->parts.size()) { + if (state->completion) { + state->completion.complete(std::move(status)); + } + return; + } + state->submit(index + 1); + }); + } +}; + // RAII aligned buffer for O_DIRECT writes via io_uring. std::vector memory // is not guaranteed IU-aligned; this wraps posix_memalign + memcpy. class AlignedIoBuf @@ -92,6 +124,35 @@ BlockAsyncPageStore::BlockAsyncPageStore(BlockPageStore *store, ::crowdb::common uint64_t BlockAsyncPageStore::submit_read(PageAddr addr, void *buf, size_t len, AsyncCompletion on_complete) { + if (len == 0 || len > std::numeric_limits::max() - addr) { + if (on_complete) { + on_complete.complete(len == 0 ? Status::Ok() : Status::invalid_argument("read range overflow")); + } + return 0; + } + const uint64_t block_size = store_->block_size(); + if (block_size > 0 && len > block_size - (addr % block_size)) { + auto state = std::make_shared(); + state->uring = uring_; + state->completion = on_complete; + size_t done = 0; + while (done < len) { + uint64_t local = 0; + int fd = store_->fd_for_offset(addr + done, &local); + if (fd < 0) { + if (on_complete) { + on_complete.complete(Status::io_error("BlockAsyncPageStore: no fd for offset")); + } + return 0; + } + size_t chunk = std::min(static_cast(len - done), block_size - local); + state->parts.push_back( + {.fd = fd, .buffer = static_cast(buf) + done, .length = chunk, .offset = local}); + done += chunk; + } + state->submit(0); + return 0; + } uint64_t local = 0; int fd = store_->fd_for_offset(addr, &local); if (fd < 0) { @@ -188,7 +249,7 @@ uint64_t BlockAsyncPageStore::submit_write(PageAddr addr, const void *buf, size_ state->pending++; uring_->submit_write(fd, write_buf, chunk, static_cast(local), [state, chunk](int res) { - if (res < 0 || static_cast(res) < chunk) { + if (res < 0 || std::cmp_less(res, chunk)) { Status s = result_to_status(res, chunk, "write"); if (state->first_error.ok()) { state->first_error = std::move(s); diff --git a/lib/crowdb-tree/src/maptable/buffer_pool.cpp b/lib/crowdb-tree/src/maptable/buffer_pool.cpp index dbcd627d0..3fa13339d 100644 --- a/lib/crowdb-tree/src/maptable/buffer_pool.cpp +++ b/lib/crowdb-tree/src/maptable/buffer_pool.cpp @@ -68,7 +68,8 @@ BufferPool::BufferPool(size_t capacity_bytes, uint32_t page_bytes, PageStore *st if (num_frames_ == 0) { num_frames_ = 1; } - arena_.assign(static_cast(num_frames_) * page_bytes_, 0); + // Misses load a complete validated frame; fresh frames are zeroed on acquisition. + arena_ = std::make_unique_for_overwrite(static_cast(num_frames_) * page_bytes_); frames_.resize(num_frames_); stats_.num_frames = num_frames_; diff --git a/lib/crowdb-tree/src/snapshot/persist.cpp b/lib/crowdb-tree/src/snapshot/persist.cpp index a8bedc3a7..9fd503382 100644 --- a/lib/crowdb-tree/src/snapshot/persist.cpp +++ b/lib/crowdb-tree/src/snapshot/persist.cpp @@ -407,7 +407,7 @@ Status Crowdbtree::prepare_snapshot_locked(PreparedSnapshot *out, std::vectorslot_count; ++slot) { - const uint64_t page_id = seg_idx * MappingTable::kSegmentSize + slot; + const uint64_t page_id = (seg_idx * MappingTable::kSegmentSize) + slot; uint64_t word = segment->slots[slot].load(std::memory_order_relaxed); if (!slot_word::is_resident(word)) { continue; @@ -517,7 +517,6 @@ Status Crowdbtree::queue_snapshot_page_locked(SnapshotPrepareContext &ctx, uint6 auto logical = static_cast(blob.size()); *addr = ctx.alloc.alloc(logical); *logical_len = logical; - blob.resize(round_up_to_iu(logical, ctx.iu), 0); ctx.out->page_writes.push_back(PreparedPageWrite{ .page_id = page_id, .page = page, @@ -1054,277 +1053,6 @@ Status Crowdbtree::materialize_ownership(uint64_t *bytes_written, bool *complete return status; } -Status Crowdbtree::snapshot(uint64_t *out_last_applied, uint64_t *out_snapshot_seq) -{ - if (opt_.page_store == nullptr) { - return Status::invalid_argument("snapshot: no page_store"); - } - auto snap_t0 = std::chrono::steady_clock::now(); - acquire_snapshot_slot(); - PreparedSnapshot prepared; - Status ps; - { - auto apply_t0 = std::chrono::steady_clock::now(); - std::scoped_lock lk(write_mutex_); - ps = prepare_snapshot_locked(&prepared); - if (metrics_.snapshot_apply_l != nullptr) { - auto ns = std::chrono::duration_cast(std::chrono::steady_clock::now() - apply_t0) - .count(); - metrics_.snapshot_apply_l->observe(static_cast(ns)); - } - } - if (!ps.ok()) { - CRB_LOG_ERROR("[{}] snapshot: prepare_snapshot_locked failed: {}", name_, ps.to_string()); - release_snapshot_slot(); - return ps; - } - for (auto &w : prepared.page_writes) { - auto write_t0 = std::chrono::steady_clock::now(); - Status s = opt_.page_store->write_at(w.addr, w.blob.data(), w.blob.size()); - if (metrics_.snapshot_page_write_l != nullptr) { - auto ns = std::chrono::duration_cast(std::chrono::steady_clock::now() - write_t0) - .count(); - metrics_.snapshot_page_write_l->observe(static_cast(ns)); - } - if (metrics_.snapshot_page_write_bw != nullptr) { - metrics_.snapshot_page_write_bw->observe(w.blob.size()); - } - if (!s.ok()) { - CRB_LOG_ERROR("[{}] snapshot page write failed: addr={} status={}", name_, w.addr, s.to_string()); - release_snapshot_slot(); - return s; - } - } - for (auto &sw : prepared.segment_writes) { - Status s = opt_.page_store->write_typed_at(PagePurpose::kPageIndex, sw.addr, sw.blob.data(), sw.blob.size()); - if (!s.ok()) { - CRB_LOG_ERROR("[{}] snapshot segment write failed: addr={} status={}", name_, sw.addr, s.to_string()); - release_snapshot_slot(); - return s; - } - } - Status dw = - opt_.page_store->write_typed_at(PagePurpose::kPageIndex, prepared.directory_write.addr, - prepared.directory_write.blob.data(), prepared.directory_write.blob.size()); - if (!dw.ok()) { - CRB_LOG_ERROR("[{}] snapshot directory write failed: addr={} status={}", name_, prepared.directory_write.addr, - dw.to_string()); - release_snapshot_slot(); - return dw; - } - // Barrier: pages + segment images + directory durable before the anchor - // that references them. - auto fsync_t0 = std::chrono::steady_clock::now(); - Status sync1 = opt_.page_store->sync(); - if (metrics_.fsync_l != nullptr) { - auto ns = - std::chrono::duration_cast(std::chrono::steady_clock::now() - fsync_t0).count(); - metrics_.fsync_l->observe(static_cast(ns)); - } - if (!sync1.ok()) { - CRB_LOG_ERROR("[{}] snapshot data sync failed: {}", name_, sync1.to_string()); - release_snapshot_slot(); - return sync1; - } - Status aw = opt_.page_store->write_typed_at(PagePurpose::kPageIndex, prepared.anchor_write.addr, - prepared.anchor_write.blob.data(), prepared.anchor_write.blob.size()); - if (!aw.ok()) { - CRB_LOG_ERROR("[{}] snapshot anchor write failed: addr={} status={}", name_, prepared.anchor_write.addr, - aw.to_string()); - release_snapshot_slot(); - return aw; - } - auto fsync2_t0 = std::chrono::steady_clock::now(); - Status sync2 = opt_.page_store->sync(); - if (metrics_.fsync_l != nullptr) { - auto ns = - std::chrono::duration_cast(std::chrono::steady_clock::now() - fsync2_t0).count(); - metrics_.fsync_l->observe(static_cast(ns)); - } - if (!sync2.ok()) { - CRB_LOG_ERROR("[{}] snapshot anchor sync failed: {}", name_, sync2.to_string()); - commit_prepared_snapshot(prepared); - release_snapshot_slot(); - return sync2; - } - - // Observe metadata write bytes: segments + directory + anchor. - if (metrics_.snapshot_meta_write_bw != nullptr) { - uint64_t meta_bytes = 0; - for (const auto &sw : prepared.segment_writes) { - meta_bytes += sw.blob.size(); - } - meta_bytes += prepared.directory_write.blob.size(); - meta_bytes += prepared.anchor_write.blob.size(); - metrics_.snapshot_meta_write_bw->observe(meta_bytes); - } - - finalize_prepared_snapshot(prepared); - - // Publish the exact logical summary only after the root anchor is durable. - if (PageBase *root = resident(root_page_id_.load(std::memory_order_acquire)); root != nullptr) { - PageSummary summary; - if (root->type == page_type::kLeafBase) { - summary = static_cast(root)->summary(); - } - else if (root->type == page_type::kInnerBase) { - summary = static_cast(root)->summary(); - } - if (summary.exact) { - summary_live_kv_.store(summary.live_kv, std::memory_order_release); - summary_live_key_bytes_.store(summary.live_key_bytes, std::memory_order_release); - summary_live_value_bytes_.store(summary.live_value_bytes, std::memory_order_release); - summary_live_logical_bytes_.store(summary.live_logical_bytes, std::memory_order_release); - summary_reachable_overflow_pages_.store(summary.reachable_overflow_pages, std::memory_order_release); - summary_reachable_page_capacity_bytes_.store(summary.reachable_page_capacity_bytes, - std::memory_order_release); - summary_covered_slot_.store(prepared.last_applied_slot, std::memory_order_release); - summary_root_version_.store(prepared.seq, std::memory_order_release); - summary_available_.store(true, std::memory_order_release); - } - } - - release_snapshot_slot(); - if (out_last_applied != nullptr) { - *out_last_applied = prepared.last_applied_slot; - } - if (out_snapshot_seq != nullptr) { - *out_snapshot_seq = prepared.seq; - } - if (metrics_.snapshot_l != nullptr) { - auto ns = - std::chrono::duration_cast(std::chrono::steady_clock::now() - snap_t0).count(); - metrics_.snapshot_l->observe(static_cast(ns)); - } - return Status::Ok(); -} - -void Crowdbtree::snapshot_async( - std::function on_done) // NOLINT(performance-unnecessary-value-param) -{ - if (opt_.page_store == nullptr) { - on_done(Status::invalid_argument("snapshot: no page_store"), 0); - return; - } - if (opt_.async_page_store != nullptr) { - acquire_snapshot_slot(); - auto prepared = std::make_shared(); - Status ps; - { - std::scoped_lock lk(write_mutex_); - ps = prepare_snapshot_locked(prepared.get()); - } - if (!ps.ok()) { - release_snapshot_slot(); - on_done(ps, 0); - return; - } - snapshot_write_next_async(std::move(prepared), 0, std::move(on_done)); - return; - } - // No async backend wired -- run the synchronous path in - // this stack frame; still correct, just not genuinely async. - uint64_t last_applied = 0; - Status st = snapshot(&last_applied); - on_done(st, last_applied); -} - -void Crowdbtree::snapshot_write_next_async( // NOLINT(readability-convert-member-functions-to-static) - std::shared_ptr prepared, // NOLINT(performance-unnecessary-value-param) - size_t idx, std::function on_done) // NOLINT(performance-unnecessary-value-param) -{ - // snapshot_inflight_ (acquired by snapshot_async() before - // prepare_snapshot_locked()) stays held across this entire async chain - // -- see snapshot_async's doc comment on crowdb-tree.h for why an atomic - // spin-gate is used here instead of write_mutex_ (which cannot be - // unlocked from a different thread than the one that locked it, and - // this chain's completions run on the Reactor thread). `prepared` is a - // shared_ptr so each hop's lambda can carry it to the next hop after - // this call's own stack frame returns. - if (idx < prepared->page_writes.size()) { - const PreparedPageWrite &w = prepared->page_writes[idx]; - opt_.async_page_store->submit_write( - w.addr, w.blob.data(), w.blob.size(), - detail::own_async_completion([this, prepared, idx, on_done](const Status &st) mutable { - if (!st.ok()) { - release_snapshot_slot(); - on_done(st, 0); - return; - } - snapshot_write_next_async(std::move(prepared), idx + 1, std::move(on_done)); - })); - return; - } - size_t seg_idx_in_list = idx - prepared->page_writes.size(); - if (seg_idx_in_list < prepared->segment_writes.size()) { - const PreparedSegmentWrite &sw = prepared->segment_writes[seg_idx_in_list]; - opt_.async_page_store->submit_typed_write( - PagePurpose::kPageIndex, sw.addr, sw.blob.data(), sw.blob.size(), - detail::own_async_completion([this, prepared, idx, on_done](const Status &st) mutable { - if (!st.ok()) { - release_snapshot_slot(); - on_done(st, 0); - return; - } - snapshot_write_next_async(std::move(prepared), idx + 1, std::move(on_done)); - })); - return; - } - - const PreparedSnapshotWrite &dw = prepared->directory_write; - opt_.async_page_store->submit_typed_write( - PagePurpose::kPageIndex, dw.addr, dw.blob.data(), dw.blob.size(), - detail::own_async_completion([this, prepared, on_done](const Status &st) mutable { - if (!st.ok()) { - release_snapshot_slot(); - on_done(st, 0); - return; - } - // Barrier: pages + segment images + directory durable before the anchor - // that references them. - Status fs1 = opt_.async_page_store->submit_fsync( - detail::own_async_completion([this, prepared, on_done](const Status &st2) mutable { - if (!st2.ok()) { - release_snapshot_slot(); - on_done(st2, 0); - return; - } - const PreparedSnapshotWrite &aw = prepared->anchor_write; - opt_.async_page_store->submit_typed_write( - PagePurpose::kPageIndex, aw.addr, aw.blob.data(), aw.blob.size(), - detail::own_async_completion([this, prepared, on_done](const Status &st3) mutable { - if (!st3.ok()) { - release_snapshot_slot(); - on_done(st3, 0); - return; - } - Status fs2 = opt_.async_page_store->submit_fsync( - detail::own_async_completion([this, prepared, on_done](const Status &st4) mutable { - if (!st4.ok()) { - commit_prepared_snapshot(*prepared); - release_snapshot_slot(); - on_done(st4, 0); - return; - } - finalize_prepared_snapshot(*prepared); - uint64_t last_applied = prepared->last_applied_slot; - release_snapshot_slot(); - on_done(Status::Ok(), last_applied); - })); - if (!fs2.ok()) { - commit_prepared_snapshot(*prepared); - release_snapshot_slot(); - on_done(fs2, 0); - } - })); - })); - if (!fs1.ok()) { - release_snapshot_slot(); - on_done(fs1, 0); - } - })); -} - Status Crowdbtree::prefetch_sparse_pages(std::vector *out, std::set *selected_blocks) { selected_blocks->clear(); @@ -1386,71 +1114,6 @@ Status Crowdbtree::prefetch_sparse_pages(std::vector *out, std:: return Status::Ok(); } -Status Crowdbtree::persist_compaction_snapshot(std::vector prefetched, - std::set selected_blocks, PreparedSnapshot *prepared) -{ - acquire_snapshot_slot(); - Status prepare; - { - std::scoped_lock lk(write_mutex_); - prepare = prepare_snapshot_locked(prepared, std::move(prefetched), std::move(selected_blocks)); - } - if (!prepare.ok()) { - CRB_LOG_ERROR("[{}] compact_sparse_blocks: prepare failed: {}", name_, prepare.to_string()); - release_snapshot_slot(); - return prepare; - } - - for (auto &write : prepared->page_writes) { - Status status = opt_.page_store->write_at(write.addr, write.blob.data(), write.blob.size()); - if (!status.ok()) { - release_snapshot_slot(); - return status; - } - } - for (auto &write : prepared->segment_writes) { - Status status = - opt_.page_store->write_typed_at(PagePurpose::kPageIndex, write.addr, write.blob.data(), write.blob.size()); - if (!status.ok()) { - release_snapshot_slot(); - return status; - } - } - Status directory = - opt_.page_store->write_typed_at(PagePurpose::kPageIndex, prepared->directory_write.addr, - prepared->directory_write.blob.data(), prepared->directory_write.blob.size()); - if (!directory.ok()) { - release_snapshot_slot(); - return directory; - } - Status durable_contents = opt_.page_store->sync(); - if (!durable_contents.ok()) { - release_snapshot_slot(); - return durable_contents; - } - Status anchor = - opt_.page_store->write_typed_at(PagePurpose::kPageIndex, prepared->anchor_write.addr, - prepared->anchor_write.blob.data(), prepared->anchor_write.blob.size()); - if (!anchor.ok()) { - release_snapshot_slot(); - return anchor; - } - Status committed = opt_.page_store->sync(); - if (!committed.ok()) { - std::vector invalid_anchor(prepared->anchor_write.blob.size(), 0); - Status rollback = opt_.page_store->write_typed_at(PagePurpose::kPageIndex, prepared->anchor_write.addr, - invalid_anchor.data(), invalid_anchor.size()); - if (rollback.ok()) { - (void)opt_.page_store->sync(); - } - release_snapshot_slot(); - return committed; - } - finalize_prepared_snapshot(*prepared); - release_snapshot_slot(); - return Status::Ok(); -} - void Crowdbtree::record_compaction_metrics(const MergeGcStats &stats, uint64_t elapsed_ns) { if (metrics_.merge_gc_blocks_c != nullptr && stats.blocks_selected > 0) { diff --git a/lib/crowdb-tree/src/snapshot/retry.cpp b/lib/crowdb-tree/src/snapshot/retry.cpp new file mode 100644 index 000000000..e83b9da85 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/retry.cpp @@ -0,0 +1,66 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "async_completion_adapter.h" +#include "crowdb-common/log.h" +#include "crowdb-tree/backend/async_page_store.h" +#include "crowdb-tree/crowdb-tree.h" + +#include +#include +#include + +namespace crowdb::tree +{ + +Status Crowdbtree::retry_snapshot_anchor() +{ + if (!snapshot_anchor_pending_) { + return Status::Ok(); + } + Status status = opt_.page_store->sync(); + if (status.ok()) { + snapshot_anchor_pending_ = false; + } + else { + CRB_LOG_ERROR("[{}] snapshot pending anchor sync failed: {}", name_, status.to_string()); + } + return status; +} + +void Crowdbtree::snapshot_prepare_async(std::function on_done) +{ + if (snapshot_anchor_pending_) { + Status submitted = opt_.async_page_store->submit_fsync( + detail::own_async_completion([this, on_done](const Status &status) mutable { + if (!status.ok()) { + CRB_LOG_ERROR("[{}] snapshot pending anchor sync failed: {}", name_, status.to_string()); + release_snapshot_slot(); + on_done(status, 0); + return; + } + snapshot_anchor_pending_ = false; + snapshot_prepare_async(std::move(on_done)); + })); + if (!submitted.ok()) { + release_snapshot_slot(); + on_done(std::move(submitted), 0); + } + return; + } + auto prepared = std::make_shared(); + Status status; + { + std::scoped_lock lock(write_mutex_); + status = prepare_snapshot_locked(prepared.get()); + } + if (!status.ok()) { + CRB_LOG_ERROR("[{}] snapshot prepare failed: {}", name_, status.to_string()); + release_snapshot_slot(); + on_done(std::move(status), 0); + return; + } + snapshot_write_next_async(std::move(prepared), 0, std::move(on_done)); +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/write.cpp b/lib/crowdb-tree/src/snapshot/write.cpp new file mode 100644 index 000000000..f0502ab6a --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/write.cpp @@ -0,0 +1,365 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "async_completion_adapter.h" +#include "crowdb-common/log.h" +#include "crowdb-tree/backend/async_page_store.h" +#include "crowdb-tree/crowdb-tree.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace crowdb::tree +{ +Status Crowdbtree::snapshot(uint64_t *out_last_applied, uint64_t *out_snapshot_seq) +{ + if (opt_.page_store == nullptr) { + return Status::invalid_argument("snapshot: no page_store"); + } + auto snap_t0 = std::chrono::steady_clock::now(); + acquire_snapshot_slot(); + Status pending = retry_snapshot_anchor(); + if (!pending.ok()) { + release_snapshot_slot(); + return pending; + } + PreparedSnapshot prepared; + Status ps; + { + auto apply_t0 = std::chrono::steady_clock::now(); + std::scoped_lock lk(write_mutex_); + ps = prepare_snapshot_locked(&prepared); + if (metrics_.snapshot_apply_l != nullptr) { + auto ns = std::chrono::duration_cast(std::chrono::steady_clock::now() - apply_t0) + .count(); + metrics_.snapshot_apply_l->observe(static_cast(ns)); + } + } + if (!ps.ok()) { + CRB_LOG_ERROR("[{}] snapshot: prepare_snapshot_locked failed: {}", name_, ps.to_string()); + release_snapshot_slot(); + return ps; + } + for (auto &w : prepared.page_writes) { + w.blob.resize(round_up_to_iu(w.logical_len, opt_.page_store->iu_size()), 0); + auto write_t0 = std::chrono::steady_clock::now(); + Status s = opt_.page_store->write_at(w.addr, w.blob.data(), w.blob.size()); + if (metrics_.snapshot_page_write_l != nullptr) { + auto ns = std::chrono::duration_cast(std::chrono::steady_clock::now() - write_t0) + .count(); + metrics_.snapshot_page_write_l->observe(static_cast(ns)); + } + if (metrics_.snapshot_page_write_bw != nullptr) { + metrics_.snapshot_page_write_bw->observe(w.blob.size()); + } + if (!s.ok()) { + CRB_LOG_ERROR("[{}] snapshot page write failed: addr={} status={}", name_, w.addr, s.to_string()); + release_snapshot_slot(); + return s; + } + std::vector().swap(w.blob); + } + for (auto &sw : prepared.segment_writes) { + Status s = opt_.page_store->write_typed_at(PagePurpose::kPageIndex, sw.addr, sw.blob.data(), sw.blob.size()); + if (!s.ok()) { + CRB_LOG_ERROR("[{}] snapshot segment write failed: addr={} status={}", name_, sw.addr, s.to_string()); + release_snapshot_slot(); + return s; + } + } + Status dw = + opt_.page_store->write_typed_at(PagePurpose::kPageIndex, prepared.directory_write.addr, + prepared.directory_write.blob.data(), prepared.directory_write.blob.size()); + if (!dw.ok()) { + CRB_LOG_ERROR("[{}] snapshot directory write failed: addr={} status={}", name_, prepared.directory_write.addr, + dw.to_string()); + release_snapshot_slot(); + return dw; + } + // Barrier: pages + segment images + directory durable before the anchor + // that references them. + auto fsync_t0 = std::chrono::steady_clock::now(); + Status sync1 = opt_.page_store->sync(); + if (metrics_.fsync_l != nullptr) { + auto ns = + std::chrono::duration_cast(std::chrono::steady_clock::now() - fsync_t0).count(); + metrics_.fsync_l->observe(static_cast(ns)); + } + if (!sync1.ok()) { + CRB_LOG_ERROR("[{}] snapshot data sync failed: {}", name_, sync1.to_string()); + release_snapshot_slot(); + return sync1; + } + Status aw = opt_.page_store->write_typed_at(PagePurpose::kPageIndex, prepared.anchor_write.addr, + prepared.anchor_write.blob.data(), prepared.anchor_write.blob.size()); + if (!aw.ok()) { + CRB_LOG_ERROR("[{}] snapshot anchor write failed: addr={} status={}", name_, prepared.anchor_write.addr, + aw.to_string()); + release_snapshot_slot(); + return aw; + } + auto fsync2_t0 = std::chrono::steady_clock::now(); + snapshot_anchor_pending_ = true; + Status sync2 = opt_.page_store->sync(); + if (metrics_.fsync_l != nullptr) { + auto ns = + std::chrono::duration_cast(std::chrono::steady_clock::now() - fsync2_t0).count(); + metrics_.fsync_l->observe(static_cast(ns)); + } + if (!sync2.ok()) { + CRB_LOG_ERROR("[{}] snapshot anchor sync failed: {}", name_, sync2.to_string()); + commit_prepared_snapshot(prepared); + release_snapshot_slot(); + return sync2; + } + snapshot_anchor_pending_ = false; + + // Observe metadata write bytes: segments + directory + anchor. + if (metrics_.snapshot_meta_write_bw != nullptr) { + uint64_t meta_bytes = 0; + for (const auto &sw : prepared.segment_writes) { + meta_bytes += sw.blob.size(); + } + meta_bytes += prepared.directory_write.blob.size(); + meta_bytes += prepared.anchor_write.blob.size(); + metrics_.snapshot_meta_write_bw->observe(meta_bytes); + } + + finalize_prepared_snapshot(prepared); + + // Publish the exact logical summary only after the root anchor is durable. + if (PageBase *root = resident(root_page_id_.load(std::memory_order_acquire)); root != nullptr) { + PageSummary summary; + if (root->type == page_type::kLeafBase) { + summary = static_cast(root)->summary(); + } + else if (root->type == page_type::kInnerBase) { + summary = static_cast(root)->summary(); + } + if (summary.exact) { + summary_live_kv_.store(summary.live_kv, std::memory_order_release); + summary_live_key_bytes_.store(summary.live_key_bytes, std::memory_order_release); + summary_live_value_bytes_.store(summary.live_value_bytes, std::memory_order_release); + summary_live_logical_bytes_.store(summary.live_logical_bytes, std::memory_order_release); + summary_reachable_overflow_pages_.store(summary.reachable_overflow_pages, std::memory_order_release); + summary_reachable_page_capacity_bytes_.store(summary.reachable_page_capacity_bytes, + std::memory_order_release); + summary_covered_slot_.store(prepared.last_applied_slot, std::memory_order_release); + summary_root_version_.store(prepared.seq, std::memory_order_release); + summary_available_.store(true, std::memory_order_release); + } + } + + release_snapshot_slot(); + if (out_last_applied != nullptr) { + *out_last_applied = prepared.last_applied_slot; + } + if (out_snapshot_seq != nullptr) { + *out_snapshot_seq = prepared.seq; + } + if (metrics_.snapshot_l != nullptr) { + auto ns = + std::chrono::duration_cast(std::chrono::steady_clock::now() - snap_t0).count(); + metrics_.snapshot_l->observe(static_cast(ns)); + } + return Status::Ok(); +} + +void Crowdbtree::snapshot_async( + std::function on_done) // NOLINT(performance-unnecessary-value-param) +{ + if (opt_.page_store == nullptr) { + on_done(Status::invalid_argument("snapshot: no page_store"), 0); + return; + } + if (opt_.async_page_store != nullptr) { + acquire_snapshot_slot(); + snapshot_prepare_async(std::move(on_done)); + return; + } + // No async backend wired -- run the synchronous path in + // this stack frame; still correct, just not genuinely async. + uint64_t last_applied = 0; + Status st = snapshot(&last_applied); + on_done(st, last_applied); +} + +void Crowdbtree::snapshot_write_next_async( // NOLINT(readability-convert-member-functions-to-static) + std::shared_ptr prepared, // NOLINT(performance-unnecessary-value-param) + size_t idx, std::function on_done) // NOLINT(performance-unnecessary-value-param) +{ + // snapshot_inflight_ (acquired by snapshot_async() before + // prepare_snapshot_locked()) stays held across this entire async chain + // -- see snapshot_async's doc comment on crowdb-tree.h for why an atomic + // spin-gate is used here instead of write_mutex_ (which cannot be + // unlocked from a different thread than the one that locked it, and + // this chain's completions run on the Reactor thread). `prepared` is a + // shared_ptr so each hop's lambda can carry it to the next hop after + // this call's own stack frame returns. + if (idx < prepared->page_writes.size()) { + PreparedPageWrite &w = prepared->page_writes[idx]; + w.blob.resize(round_up_to_iu(w.logical_len, opt_.page_store->iu_size()), 0); + opt_.async_page_store->submit_write( + w.addr, w.blob.data(), w.blob.size(), + detail::own_async_completion([this, prepared, idx, on_done](const Status &st) mutable { + if (!st.ok()) { + release_snapshot_slot(); + on_done(st, 0); + return; + } + std::vector().swap(prepared->page_writes[idx].blob); + snapshot_write_next_async(std::move(prepared), idx + 1, std::move(on_done)); + })); + return; + } + size_t seg_idx_in_list = idx - prepared->page_writes.size(); + if (seg_idx_in_list < prepared->segment_writes.size()) { + const PreparedSegmentWrite &sw = prepared->segment_writes[seg_idx_in_list]; + opt_.async_page_store->submit_typed_write( + PagePurpose::kPageIndex, sw.addr, sw.blob.data(), sw.blob.size(), + detail::own_async_completion([this, prepared, idx, on_done](const Status &st) mutable { + if (!st.ok()) { + release_snapshot_slot(); + on_done(st, 0); + return; + } + snapshot_write_next_async(std::move(prepared), idx + 1, std::move(on_done)); + })); + return; + } + + const PreparedSnapshotWrite &dw = prepared->directory_write; + opt_.async_page_store->submit_typed_write( + PagePurpose::kPageIndex, dw.addr, dw.blob.data(), dw.blob.size(), + detail::own_async_completion([this, prepared, on_done](const Status &st) mutable { + if (!st.ok()) { + release_snapshot_slot(); + on_done(st, 0); + return; + } + // Barrier: pages + segment images + directory durable before the anchor + // that references them. + Status fs1 = opt_.async_page_store->submit_fsync( + detail::own_async_completion([this, prepared, on_done](const Status &st2) mutable { + if (!st2.ok()) { + release_snapshot_slot(); + on_done(st2, 0); + return; + } + const PreparedSnapshotWrite &aw = prepared->anchor_write; + opt_.async_page_store->submit_typed_write( + PagePurpose::kPageIndex, aw.addr, aw.blob.data(), aw.blob.size(), + detail::own_async_completion([this, prepared, on_done](const Status &st3) mutable { + if (!st3.ok()) { + release_snapshot_slot(); + on_done(st3, 0); + return; + } + snapshot_anchor_pending_ = true; + Status fs2 = opt_.async_page_store->submit_fsync( + detail::own_async_completion([this, prepared, on_done](const Status &st4) mutable { + if (!st4.ok()) { + CRB_LOG_ERROR("[{}] snapshot anchor sync failed: {}", name_, st4.to_string()); + commit_prepared_snapshot(*prepared); + release_snapshot_slot(); + on_done(st4, 0); + return; + } + snapshot_anchor_pending_ = false; + finalize_prepared_snapshot(*prepared); + uint64_t last_applied = prepared->last_applied_slot; + release_snapshot_slot(); + on_done(Status::Ok(), last_applied); + })); + if (!fs2.ok()) { + commit_prepared_snapshot(*prepared); + release_snapshot_slot(); + on_done(fs2, 0); + } + })); + })); + if (!fs1.ok()) { + release_snapshot_slot(); + on_done(fs1, 0); + } + })); +} + +Status Crowdbtree::persist_compaction_snapshot(std::vector prefetched, + std::set selected_blocks, PreparedSnapshot *prepared) +{ + acquire_snapshot_slot(); + Status pending = retry_snapshot_anchor(); + if (!pending.ok()) { + release_snapshot_slot(); + return pending; + } + Status prepare; + { + std::scoped_lock lk(write_mutex_); + prepare = prepare_snapshot_locked(prepared, std::move(prefetched), std::move(selected_blocks)); + } + if (!prepare.ok()) { + CRB_LOG_ERROR("[{}] compact_sparse_blocks: prepare failed: {}", name_, prepare.to_string()); + release_snapshot_slot(); + return prepare; + } + + for (auto &write : prepared->page_writes) { + write.blob.resize(round_up_to_iu(write.logical_len, opt_.page_store->iu_size()), 0); + Status status = opt_.page_store->write_at(write.addr, write.blob.data(), write.blob.size()); + if (!status.ok()) { + release_snapshot_slot(); + return status; + } + std::vector().swap(write.blob); + } + for (auto &write : prepared->segment_writes) { + Status status = + opt_.page_store->write_typed_at(PagePurpose::kPageIndex, write.addr, write.blob.data(), write.blob.size()); + if (!status.ok()) { + release_snapshot_slot(); + return status; + } + } + Status directory = + opt_.page_store->write_typed_at(PagePurpose::kPageIndex, prepared->directory_write.addr, + prepared->directory_write.blob.data(), prepared->directory_write.blob.size()); + if (!directory.ok()) { + release_snapshot_slot(); + return directory; + } + Status durable_contents = opt_.page_store->sync(); + if (!durable_contents.ok()) { + release_snapshot_slot(); + return durable_contents; + } + Status anchor = + opt_.page_store->write_typed_at(PagePurpose::kPageIndex, prepared->anchor_write.addr, + prepared->anchor_write.blob.data(), prepared->anchor_write.blob.size()); + if (!anchor.ok()) { + release_snapshot_slot(); + return anchor; + } + Status committed = opt_.page_store->sync(); + if (!committed.ok()) { + std::vector invalid_anchor(prepared->anchor_write.blob.size(), 0); + Status rollback = opt_.page_store->write_typed_at(PagePurpose::kPageIndex, prepared->anchor_write.addr, + invalid_anchor.data(), invalid_anchor.size()); + if (rollback.ok()) { + (void)opt_.page_store->sync(); + } + release_snapshot_slot(); + return committed; + } + finalize_prepared_snapshot(*prepared); + release_snapshot_slot(); + return Status::Ok(); +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/chunk_page_staging_test.cpp b/lib/crowdb-tree/tests/integration/chunk_page_staging_test.cpp new file mode 100644 index 000000000..fb64a3d2a --- /dev/null +++ b/lib/crowdb-tree/tests/integration/chunk_page_staging_test.cpp @@ -0,0 +1,90 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "backend/chunk/chunk_page_store.h" + +#include + +#include +#include +#include +#include + +namespace crowdb::tree::detail +{ +namespace +{ +class ChunkPageStagingTest : public ::testing::TestWithParam +{ +}; + +void publish(ChunkPageStore &store, bool asynchronous) +{ + ASSERT_TRUE(store.sync().ok()); + const uint8_t anchor = 7; + ASSERT_TRUE(store.write_at(0, &anchor, 1).ok()); + if (!asynchronous) { + ASSERT_TRUE(store.sync().ok()); + return; + } + + struct Completion + { + std::atomic done{false}; + bool success = false; + } completion; + + ASSERT_TRUE(store + .submit_fsync({.context = &completion, + .complete_fn = + [](void *context, Status status) { + auto &result = *static_cast(context); + result.success = status.ok(); + result.done.store(true, std::memory_order_release); + result.done.notify_one(); + }}) + .ok()); + completion.done.wait(false, std::memory_order_acquire); + ASSERT_TRUE(completion.success); +} + +TEST_P(ChunkPageStagingTest, SmallOverwriteDoesNotMaterializeThePublishedImage) +{ + auto catalog = std::make_shared(1); + auto transport = std::make_shared(); + const ChunkPageStore::Config config{.tree_id = 921, .owner_epoch = 1, .pack_bytes = 65536, .iu_size = 1}; + std::vector expected(16 * 1024 * 1024, 0x53); + { + ChunkPageStore store(config, catalog, transport); + ASSERT_TRUE(store.write_at(8192, expected.data(), expected.size()).ok()); + publish(store, GetParam()); + } + { + ChunkPageStore store(config, catalog, transport); + const std::array first{1, 2, 3, 4}; + const std::array last{8, 9}; + ASSERT_TRUE(store.write_at(8192 + 65535, first.data(), first.size()).ok()); + ASSERT_TRUE(store.write_at(8192 + 65536, last.data(), last.size()).ok()); + ASSERT_TRUE(store.write_at(131071, first.data(), first.size()).ok()); + ASSERT_TRUE(store.write_at(131072, last.data(), last.size()).ok()); + EXPECT_LE(store.stats().pack_reads, 2U) + << "small staging writes must not read and retain the complete published image"; + expected[65535] = 1; + expected[65536] = 8; + expected[65537] = 9; + expected[65538] = 4; + expected[131071 - 8192] = 1; + expected[131072 - 8192] = 8; + expected[131073 - 8192] = 9; + expected[131074 - 8192] = 4; + publish(store, GetParam()); + } + ChunkPageStore reopened(config, catalog, transport); + std::vector actual(expected.size()); + ASSERT_TRUE(reopened.read_at(8192, actual.data(), actual.size()).ok()); + EXPECT_EQ(actual, expected); +} + +INSTANTIATE_TEST_SUITE_P(PublicationPaths, ChunkPageStagingTest, ::testing::Bool()); +} // namespace +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/integration/chunk_page_store_test.cpp b/lib/crowdb-tree/tests/integration/chunk_page_store_test.cpp index 86de65b90..5d7d84dcf 100644 --- a/lib/crowdb-tree/tests/integration/chunk_page_store_test.cpp +++ b/lib/crowdb-tree/tests/integration/chunk_page_store_test.cpp @@ -176,7 +176,7 @@ class BlockingWriteTransport final : public ChunkTransport uint32_t observed = max_active_writes_.load(std::memory_order_relaxed); while (observed < active && !max_active_writes_.compare_exchange_weak(observed, active)) { } - active_writes_.notify_all(); + max_active_writes_.notify_all(); if (hold_writes_.load(std::memory_order_acquire)) { release_writes_.wait(false, std::memory_order_acquire); } @@ -213,10 +213,11 @@ class BlockingWriteTransport final : public ChunkTransport void wait_for_concurrent_writes(uint32_t count) { - uint32_t active = active_writes_.load(std::memory_order_acquire); - while (active < count) { - active_writes_.wait(active, std::memory_order_acquire); - active = active_writes_.load(std::memory_order_acquire); + // An active count is visible before its corresponding maximum is published. + uint32_t observed = max_active_writes_.load(std::memory_order_acquire); + while (observed < count) { + max_active_writes_.wait(observed, std::memory_order_acquire); + observed = max_active_writes_.load(std::memory_order_acquire); } } diff --git a/lib/crowdb-tree/tests/integration/chunk_snapshot_retry_test.cpp b/lib/crowdb-tree/tests/integration/chunk_snapshot_retry_test.cpp new file mode 100644 index 000000000..e9fb5ea6e --- /dev/null +++ b/lib/crowdb-tree/tests/integration/chunk_snapshot_retry_test.cpp @@ -0,0 +1,115 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "backend/chunk/chunk_page_store.h" +#include "crowdb-tree/crowdb-tree.h" + +#include + +#include +#include +#include +#include + +namespace crowdb::tree::detail +{ +namespace +{ + +Status snapshot(Crowdbtree &tree, bool asynchronous) +{ + if (!asynchronous) { + return tree.snapshot(); + } + Status result; + std::atomic done{false}; + tree.snapshot_async([&](Status status, uint64_t /*unused*/) { + result = std::move(status); + done.store(true, std::memory_order_release); + done.notify_one(); + }); + done.wait(false, std::memory_order_acquire); + return result; +} + +void check_snapshot_retry(bool asynchronous, bool write_while_unavailable = false) +{ + auto catalog = std::make_shared(1); + auto transport = std::make_shared(); + ChunkPageStore store({.tree_id = 201, .owner_epoch = 1, .pack_bytes = 4096, .iu_size = 1}, catalog, transport); + Config options; + options.page_store = &store; + options.async_page_store = asynchronous ? &store : nullptr; + Crowdbtree tree(options); + ASSERT_TRUE(tree.apply(1, Batch{{{"key", OpKind::kPut, "old"}}}).ok()); + ASSERT_TRUE(tree.flush().ok()); + ASSERT_TRUE(snapshot(tree, asynchronous).ok()); + auto prior = catalog->load(201); + ASSERT_NE(prior, nullptr); + + ASSERT_TRUE(tree.apply(2, Batch{{{"key", OpKind::kPut, "new"}}}).ok()); + ASSERT_TRUE(tree.flush().ok()); + store.inject_mirror_write_failures(1U << 1U); + const Status failed = snapshot(tree, asynchronous); + ASSERT_EQ(failed.code(), Code::kUnavailable) << failed.to_string(); + ASSERT_EQ(catalog->load(201)->generation, prior->generation); + { + ChunkPageStore durable_store({.tree_id = 201, .owner_epoch = 1, .pack_bytes = 4096, .iu_size = 1}, catalog, + transport); + Config durable_options; + durable_options.page_store = &durable_store; + std::unique_ptr durable; + ASSERT_TRUE(Crowdbtree::open(durable_options, &durable).ok()); + std::string value; + uint64_t slot = 0; + ASSERT_TRUE(durable->get(Slice("key"), &slot, &value)); + EXPECT_EQ(slot, 1U); + EXPECT_EQ(value, "old"); + } + if (write_while_unavailable) { + ASSERT_TRUE(tree.apply(3, Batch{{{"key", OpKind::kPut, "newest"}}}).ok()); + ASSERT_TRUE(tree.flush().ok()); + } + const Status still_unavailable = snapshot(tree, asynchronous); + ASSERT_EQ(still_unavailable.code(), Code::kUnavailable) << still_unavailable.to_string(); + ASSERT_EQ(catalog->load(201)->generation, prior->generation); + store.inject_mirror_write_failures(0); + + const Status retried = snapshot(tree, asynchronous); + ASSERT_TRUE(retried.ok()) << retried.to_string(); + ASSERT_GT(catalog->load(201)->generation, prior->generation); + ChunkPageStore reopened_store({.tree_id = 201, .owner_epoch = 1, .pack_bytes = 4096, .iu_size = 1}, catalog, + transport); + options.page_store = &reopened_store; + options.async_page_store = asynchronous ? &reopened_store : nullptr; + std::unique_ptr reopened; + ASSERT_TRUE(Crowdbtree::open(options, &reopened).ok()); + std::string value; + uint64_t slot = 0; + ASSERT_TRUE(reopened->get(Slice("key"), &slot, &value)); + EXPECT_EQ(slot, write_while_unavailable ? 3U : 2U); + EXPECT_EQ(value, write_while_unavailable ? "newest" : "new"); +} + +TEST(ChunkSnapshotRetry, PublicationFailureDoesNotPoisonSynchronousSnapshotRetry) +{ + check_snapshot_retry(false); +} + +TEST(ChunkSnapshotRetry, PublicationFailureDoesNotPoisonAsynchronousSnapshotRetry) +{ + check_snapshot_retry(true); +} + +TEST(ChunkSnapshotRetry, SynchronousRetryPreservesWritesAfterFailedPublication) +{ + check_snapshot_retry(false, true); +} + +TEST(ChunkSnapshotRetry, AsynchronousRetryPreservesWritesAfterFailedPublication) +{ + check_snapshot_retry(true, true); +} + +} // namespace +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/integration/snapshot_memory_test.cpp b/lib/crowdb-tree/tests/integration/snapshot_memory_test.cpp new file mode 100644 index 000000000..3a6ff5a41 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/snapshot_memory_test.cpp @@ -0,0 +1,274 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "backend/chunk/chunk_page_staging.h" +#include "crowdb-tree/backend/async_page_store.h" +#include "crowdb-tree/backend/page_store.h" +#include "crowdb-tree/crowdb-tree.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__GLIBC__) && !defined(__SANITIZE_ADDRESS__) +# include +# include + +namespace crowdb::tree +{ +namespace +{ +size_t resident_bytes() +{ + malloc_trim(0); + std::ifstream status("/proc/self/statm"); + size_t virtual_pages = 0; + size_t resident_pages = 0; + status >> virtual_pages >> resident_pages; + return resident_pages * static_cast(sysconf(_SC_PAGESIZE)); +} + +// Reserve the device before sampling allocations. Backend growth must not be +// confused with encoded snapshot pages retained after their writes complete. +class TestSnapshotBufferStore final : public PageStore, public AsyncPageStore +{ + public: + TestSnapshotBufferStore() : bytes_(96 * 1024 * 1024) + { + } + + Status write_at(uint64_t off, const uint8_t *buf, size_t len) override + { + if (baseline > 0 && !prepared_bytes.has_value()) { + const auto current = resident_bytes(); + prepared_bytes = current > baseline ? current - baseline : 0; + } + if (off > bytes_.size() || len > bytes_.size() - off) { + return Status::resource_exhausted("test device is full"); + } + std::memcpy(bytes_.data() + off, buf, len); + size_ = std::max(size_, off + len); + return Status::Ok(); + } + + Status write_typed_at(PagePurpose /*purpose*/, uint64_t off, const uint8_t *buf, size_t len) override + { + if (!retained_bytes.has_value()) { + const auto current = resident_bytes(); + retained_bytes = current > baseline ? current - baseline : 0; + } + return write_at(off, buf, len); + } + + Status read_at(uint64_t off, uint8_t *buf, size_t len) const override + { + if (off > size_ || len > size_ - off) { + return Status::not_found("test device range is absent"); + } + std::memcpy(buf, bytes_.data() + off, len); + return Status::Ok(); + } + + Status sync() override + { + return Status::Ok(); + } + + [[nodiscard]] uint64_t size() const override + { + return size_; + } + + [[nodiscard]] uint32_t iu_size() const override + { + return 65536; + } + + uint64_t submit_read(PageAddr addr, void *buf, size_t len, AsyncCompletion done) override + { + done.complete(read_at(addr, static_cast(buf), len)); + return 0; + } + + uint64_t submit_write(PageAddr addr, const void *buf, size_t len, AsyncCompletion done) override + { + pending_ = Pending{.addr = addr, .data = buf, .length = len, .done = done, .typed = false}; + return 0; + } + + uint64_t submit_typed_write(PagePurpose /*purpose*/, PageAddr addr, const void *buf, size_t len, + AsyncCompletion done) override + { + pending_ = Pending{.addr = addr, .data = buf, .length = len, .done = done, .typed = true}; + return 0; + } + + Status submit_fsync(AsyncCompletion done) override + { + done.complete(sync()); + return Status::Ok(); + } + + void cancel(uint64_t /*operation_id*/) override + { + } + + bool complete_next() + { + if (!pending_.has_value()) { + return false; + } + const auto write = pending_.value(); + pending_.reset(); + const auto *data = static_cast(write.data); + auto status = write.typed ? write_typed_at(PagePurpose::kPageIndex, write.addr, data, write.length) + : write_at(write.addr, data, write.length); + write.done.complete(std::move(status)); + return true; + } + + size_t baseline = 0; + std::optional prepared_bytes; + std::optional retained_bytes; + + private: + struct Pending + { + uint64_t addr; + const void *data; + size_t length; + AsyncCompletion done; + bool typed; + }; + + std::vector bytes_; + uint64_t size_ = 0; + std::optional pending_; +}; + +class SnapshotMemoryTest : public ::testing::TestWithParam +{ +}; + +TEST_P(SnapshotMemoryTest, ReleasesCompletedPageBuffersBeforeWritingMetadata) +{ + TestSnapshotBufferStore store; + Config config; + config.page_store = &store; + config.async_page_store = GetParam() ? &store : nullptr; + config.frame_bytes = 65536; + config.compression = compress_algo::kNone; + const std::string value(65536, 's'); + { + Crowdbtree tree(config); + for (uint64_t i = 0; i < 256; ++i) { + const Batch batch{{{.key = "snapshot-" + std::to_string(i), .kind = OpKind::kPut, .value = value}}}; + ASSERT_TRUE(tree.apply(i + 1, batch).ok()); + } + ASSERT_TRUE(tree.flush().ok()); + store.baseline = resident_bytes(); + ASSERT_GT(store.baseline, 0U); + if (GetParam()) { + bool complete = false; + tree.snapshot_async([&](Status status, uint64_t applied) { + EXPECT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(applied, 256U); + complete = true; + }); + while (!complete) { + ASSERT_TRUE(store.complete_next()); + } + } + else { + ASSERT_TRUE(tree.snapshot().ok()); + } + ASSERT_TRUE(store.retained_bytes.has_value()); + ASSERT_TRUE(store.prepared_bytes.has_value()); + const auto prepared_bytes = store.prepared_bytes.value_or(std::numeric_limits::max()); + std::cout << "Snapshot prepared RSS growth: " << prepared_bytes << " bytes\n"; + EXPECT_LT(prepared_bytes, 48U * 1024 * 1024) << "snapshot preparation retained device padding for every page"; + const auto retained_bytes = store.retained_bytes.value_or(std::numeric_limits::max()); + std::cout << "Snapshot retained RSS growth: " << retained_bytes << " bytes\n"; + EXPECT_LT(retained_bytes, 8U * 1024 * 1024) << "completed page writes still retain their encoded buffers"; + } + config.async_page_store = nullptr; + std::unique_ptr reopened; + ASSERT_TRUE(Crowdbtree::open(config, &reopened).ok()); + for (uint64_t i = 0; i < 256; ++i) { + std::string actual; + uint64_t revision = 0; + ASSERT_TRUE(reopened->get(Slice("snapshot-" + std::to_string(i)), &revision, &actual)); + EXPECT_EQ(actual, value); + EXPECT_EQ(revision, i + 1); + } +} + +INSTANTIATE_TEST_SUITE_P(WritePaths, SnapshotMemoryTest, ::testing::Bool()); + +TEST(SnapshotMemory, PaddedOverlayKeepsZeroCoverageWithoutRetainingPadding) +{ + constexpr size_t fragment_bytes = 65536; + constexpr size_t page_count = 256; + std::vector page(2 * fragment_bytes, 0); + std::fill_n(page.begin(), fragment_bytes + 32, 0x53); + detail::ChunkPageStaging staging; + const auto before = resident_bytes(); + for (size_t i = 0; i < page_count; ++i) { + staging.write(i * page.size(), page); + } + const auto growth = resident_bytes() - before; + std::cout << "Padded overlay RSS growth: " << growth << " bytes\n"; + EXPECT_LT(growth, 24U * 1024 * 1024) << "overlay retained allocation-unit padding alongside logical page bytes"; + const size_t size = page_count * page.size(); + ASSERT_EQ(staging.size(), size); + ASSERT_TRUE(staging.covers(0, size)); + + std::vector expected(size); + for (size_t i = 0; i < page_count; ++i) { + std::copy(page.begin(), page.end(), expected.begin() + static_cast(i * page.size())); + } + const std::array patch{4, 0, 5}; + for (const size_t offset : {fragment_bytes + 1024, (3 * fragment_bytes) - 1}) { + staging.write(offset, patch); + std::copy(patch.begin(), patch.end(), expected.begin() + static_cast(offset)); + } + const std::vector zeros(fragment_bytes, 0); + staging.write(0, zeros); + std::fill_n(expected.begin(), fragment_bytes, 0); + std::vector actual(size, 0xAA); + staging.overlay(0, actual); + EXPECT_EQ(actual, expected); + EXPECT_TRUE(staging.covers(0, size)); +} + +TEST(SnapshotMemory, UnusedFrameCapacityDoesNotPrefaultTheArena) +{ + MemPageStore store(65536); + const auto before = resident_bytes(); + BufferPool pool(256U * 1024 * 1024, 65536, &store); + const auto current = resident_bytes(); + const auto growth = current > before ? current - before : 0; + std::cout << "Unused frame-cache RSS growth: " << growth << " bytes\n"; + EXPECT_LT(growth, 8U * 1024 * 1024); + EXPECT_EQ(pool.stats().num_frames, 4096U); + EXPECT_EQ(pool.stats().used, 0U); + + uint32_t index = 0; + uint8_t *bytes = nullptr; + ASSERT_TRUE(pool.acquire_frame(&index, &bytes).ok()); + EXPECT_TRUE(std::all_of(bytes, bytes + 65536, [](uint8_t byte) { return byte == 0; })); + std::memset(bytes, 0x53, 65536); + pool.release_frame(index); + EXPECT_EQ(pool.stats().num_frames, 4096U); +} +} // namespace +} // namespace crowdb::tree +#endif diff --git a/lib/crowdb-tree/tests/unit/block_async_page_store_test.cpp b/lib/crowdb-tree/tests/unit/block_async_page_store_test.cpp new file mode 100644 index 000000000..0a386cd0d --- /dev/null +++ b/lib/crowdb-tree/tests/unit/block_async_page_store_test.cpp @@ -0,0 +1,135 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "async_completion_adapter.h" +#include "crowdb-common/diskio_uring.h" +#include "crowdb-tree/backend/async_page_store.h" +#include "crowdb-tree/backend/block_page_store.h" +#include "test_tmp.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace crowdb::tree; + +namespace +{ +crowdb::common::DiskIOUring make_uring() +{ + crowdb::common::Topology topology; + crowdb::common::PipelineConfig pipeline; + pipeline.entries = 256; + pipeline.mode = crowdb::common::PollingMode::Hybrid; + topology.pipelines.push_back(pipeline); + return crowdb::common::DiskIOUring(std::move(topology)); +} + +bool wait_for(const std::atomic &completed) +{ + for (unsigned i = 0; i < 200; ++i) { + if (completed.load(std::memory_order_acquire) != 0) { + return true; + } + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + return completed.load(std::memory_order_acquire) != 0; +} +} // namespace + +TEST(BlockAsyncPageStore, CrossExtentReadAfterReopenMatchesSync) +{ + struct ReadRange + { + uint64_t block_size; + uint64_t offset; + size_t length; + }; + + for (const auto range : { + ReadRange{.block_size = 64 * 1024 * 1024, .offset = 133259264, .length = 1212416}, + ReadRange{.block_size = 128 * 1024 * 1024, .offset = 133259264, .length = 1212416}, + ReadRange{.block_size = 8192, .offset = 4096, .length = 16384 }, + ReadRange{.block_size = 8192, .offset = 8192, .length = 12288 } + }) { + SCOPED_TRACE(::testing::Message() << range.block_size << ':' << range.offset << '+' << range.length); + crowdb::tree_test::TempDir directory; + auto uring = make_uring(); + std::unique_ptr store; + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 0, 0, range.block_size, 4096, &store).ok()); + std::vector expected(range.length); + for (size_t i = 0; i < expected.size(); ++i) { + expected[i] = static_cast(((i / 4096) + i) % 251); + } + ASSERT_TRUE(store->write_at(range.offset, expected.data(), expected.size()).ok()); + ASSERT_TRUE(store->sync().ok()); + store.reset(); + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 0, 0, range.block_size, 4096, &store).ok()); + for (int fd : store->all_extent_fds()) { + uring.register_fd(fd); + } + std::unique_ptr output(std::aligned_alloc(4096, range.length), &std::free); + ASSERT_NE(output, nullptr); + ASSERT_TRUE(store->read_at(range.offset, static_cast(output.get()), range.length).ok()); + ASSERT_TRUE(std::equal(expected.begin(), expected.end(), static_cast(output.get()))); + std::fill_n(static_cast(output.get()), range.length, 0); + BlockAsyncPageStore asynchronous(store.get(), &uring); + std::atomic completed{0}; + Status status; + asynchronous.submit_read(range.offset, output.get(), range.length, + crowdb::tree::detail::own_async_completion([&](Status result) { + status = std::move(result); + completed.fetch_add(1, std::memory_order_release); + })); + ASSERT_TRUE(wait_for(completed)); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(completed.load(), 1U); + EXPECT_TRUE(std::equal(expected.begin(), expected.end(), static_cast(output.get()))); + } +} + +TEST(BlockAsyncPageStore, CrossExtentReadRejectsMissingAndTruncatedTail) +{ + for (bool missing : {false, true}) { + SCOPED_TRACE(missing); + crowdb::tree_test::TempDir directory; + auto uring = make_uring(); + std::unique_ptr store; + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 0, 0, 8192, 4096, &store).ok()); + std::vector expected(12288, 0xA5); + ASSERT_TRUE(store->write_at(4096, expected.data(), expected.size()).ok()); + if (missing) { + ASSERT_TRUE(store->delete_block(1).ok()); + } + else { + uint64_t local = 0; + ASSERT_EQ(::ftruncate(store->fd_for_offset(8192, &local), 4096), 0); + } + for (int fd : store->all_extent_fds()) { + uring.register_fd(fd); + } + std::unique_ptr output(std::aligned_alloc(4096, expected.size()), &std::free); + ASSERT_NE(output, nullptr); + BlockAsyncPageStore asynchronous(store.get(), &uring); + std::atomic completed{0}; + Status status; + asynchronous.submit_read(4096, output.get(), expected.size(), + crowdb::tree::detail::own_async_completion([&](Status result) { + status = std::move(result); + completed.fetch_add(1, std::memory_order_release); + })); + ASSERT_TRUE(wait_for(completed)); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find(missing ? "no fd for offset" : "short read (4096 of 8192 bytes)"), + std::string::npos); + EXPECT_EQ(completed.load(), 1U); + } +} diff --git a/pixi.toml b/pixi.toml index aa871e432..dc8234321 100644 --- a/pixi.toml +++ b/pixi.toml @@ -130,6 +130,8 @@ auditwheel = ">=6, <7" install-deps = { cmd = "bash tools/pixi-tasks/install-deps.sh" } # Stop services and remove build products/runtime state; --runtime-only preserves build products and persistent clusters. clean = { cmd = "bash tools/pixi-tasks/clean-build.sh" } +# Build the native libraries used by the C++ and storage component gates. +build-native = "bash tools/pixi-tasks/build-cpp.sh" # Build C++ libraries, Rust release binaries, frontend and workspace test targets. build = { cmd = "bash tools/pixi-tasks/build-release.sh", depends-on = ["install-deps"] } default = { depends-on = ["build"] } @@ -139,9 +141,9 @@ default = { depends-on = ["build"] } ################################################################################ test = "python tools/pixi-tasks/test.py" test-suite = { cmd = "python tools/pixi-tasks/test.py all", depends-on = ["build"] } -test-cpp = { cmd = "python tools/pixi-tasks/test.py unit cpp", depends-on = ["build"] } +test-cpp = { cmd = "python tools/pixi-tasks/test.py unit cpp", depends-on = ["build-native"] } test-core = { cmd = "python tools/pixi-tasks/test.py unit core" } -test-storage = { cmd = "python tools/pixi-tasks/test.py unit storage" } +test-storage = { cmd = "python tools/pixi-tasks/test.py unit storage", depends-on = ["build-native"] } test-access = { cmd = "python tools/pixi-tasks/test.py unit access" } test-console = { cmd = "python tools/pixi-tasks/test.py unit console" } test-console-ui = { cmd = "python tools/pixi-tasks/test.py unit ui", depends-on = ["install-deps"] } @@ -206,8 +208,9 @@ oci-run-1c = "python container/oci-image/docker-local.py run --nodes 1" oci-run-3c = "python container/oci-image/docker-local.py run --nodes 3" # Stop defaults to --cluster local; --all selects every recorded local cluster. oci-stop = "python container/oci-image/docker-local.py stop" -# Clean defaults to all recorded clusters; --cluster narrows the scope. -# Remove owned containers, bridges and data volumes; --keep-data preserves data. +# Clean stopped clusters and unused Docker containers, networks, volumes and build cache. +# Running clusters are preserved; --cluster/--all explicitly stop selected clusters. +# --keep-data preserves volumes, node data and restart receipts. oci-clean = "python container/oci-image/clean.py" ################################################################################ diff --git a/tools/README.md b/tools/README.md index dbe6614b0..4621f1926 100644 --- a/tools/README.md +++ b/tools/README.md @@ -127,5 +127,5 @@ pixi run test profile containerd-cluster pixi run test-e2e-iceberg ``` -On a fresh checkout, `test-cpp` depends on the unified build and `test-console-ui` -depends on dependency installation before dispatch. +On a fresh checkout, `test-cpp` and `test-storage` depend on the native library +build, while `test-console-ui` depends on dependency installation before dispatch. diff --git a/tools/pixi-tasks/install-deps.sh b/tools/pixi-tasks/install-deps.sh index 07e2be34c..fa2d59efc 100644 --- a/tools/pixi-tasks/install-deps.sh +++ b/tools/pixi-tasks/install-deps.sh @@ -6,15 +6,15 @@ cd "${PIXI_PROJECT_ROOT:?}" git config core.hooksPath .githooks bashrc="${HOME:?}/.bashrc" -if [[ ! -f "$bashrc" ]] || ! grep -Eq '^[[:space:]]*(source|\.)[[:space:]]+<\(pixi completion (--shell[ =]|-s[ =])bash\)' "$bashrc"; then +if [[ ! -f "$bashrc" ]] || ! grep -Eq 'pixi[[:space:]]+completion([^#]*[[:space:]])bash([[:space:]]|\)|$)' "$bashrc"; then cat >> "$bashrc" <<'EOF' -# Complete Pixi commands and workspace task names. +# CROWDB Pixi Bash completion: commands and workspace task names. if command -v pixi >/dev/null 2>&1; then source <(pixi completion --shell bash) fi EOF - echo "[install-deps] enabled Pixi Bash completion in $bashrc; open a new terminal to use it" + echo "[install-deps] enabled Pixi Bash completion in $bashrc; open a new Bash session or run 'source $bashrc'" fi if [[ $(uname -sm) == 'Linux x86_64' ]]; then pixi run --manifest-path container/oci-image/pixi.toml python container/oci-image/preflight.py --setup-apparmor diff --git a/tools/test-metrics/measure.py b/tools/test-metrics/measure.py index d3776c5aa..fb5a546d4 100644 --- a/tools/test-metrics/measure.py +++ b/tools/test-metrics/measure.py @@ -99,13 +99,14 @@ def main(): print(text, flush=True) raise SystemExit(f"{suite} succeeded without canonical container results") passed = sum(case["status"] == "pass" for case in cases) - failed = sum(case["status"] != "pass" for case in cases) + failed = sum(case["status"] not in ("pass", "not_executed") for case in cases) ignored, ignored_cases = 0, [] record = dict(suite=suite, environment=environment, passed=passed, ignored=ignored, ignored_cases=ignored_cases, failed=failed, seconds=elapsed, exit_code=result.returncode, log=str(log), run_started_at=run_date.isoformat()) if is_container: + record["not_executed"] = sum(case["status"] == "not_executed" for case in cases) record["scenario_seconds"] = sum(report["timings"].get("scenarios_wall", 0) for report in reports) record["container_artifacts"] = str(container_root) results.append(record)