diff --git a/.agents/skills/console-ui-e2e/SKILL.md b/.agents/skills/console-ui-e2e/SKILL.md index c04404cab..37a1fc056 100644 --- a/.agents/skills/console-ui-e2e/SKILL.md +++ b/.agents/skills/console-ui-e2e/SKILL.md @@ -59,7 +59,7 @@ when sequencing suites or CI jobs. Fix a failure with its affected spec/group, then resume the remaining selection; use this skill for browser diagnosis. ```sh -pixi run clean-env +pixi run clean --runtime-only pixi run test-container-e2e --scope console.operations --case --concurrency 1 --node-cpus 0.25 --cpus 8 --memory-mib 16384 ``` diff --git a/.agents/skills/debug-test/SKILL.md b/.agents/skills/debug-test/SKILL.md index 31df09af7..76d865870 100644 --- a/.agents/skills/debug-test/SKILL.md +++ b/.agents/skills/debug-test/SKILL.md @@ -34,7 +34,7 @@ Use existing runner records where available. - Complete shared build prerequisites before parallel test jobs. Never run concurrent builders against the same output directory; use independent output directories when parallel builds are necessary. Perform shared - environment cleanup before dispatch; do not repeat `clean-env` while other + environment cleanup before dispatch; do not repeat `clean --runtime-only` while other local jobs still own running fixtures. Host browser fixtures that require stable Docker networks also share that constraint across local CI jobs; keep their browser phases separate from other jobs' bridge creation/removal. diff --git a/.agents/skills/implement-requirement/SKILL.md b/.agents/skills/implement-requirement/SKILL.md index c7838289d..c2dda3418 100644 --- a/.agents/skills/implement-requirement/SKILL.md +++ b/.agents/skills/implement-requirement/SKILL.md @@ -25,7 +25,7 @@ design and testable contract; do not create a separate design draft. commit. Find tests with `pixi task list`; prefix server-spawning tests with -`pixi run clean-env &&`. Before push run fmt and `pixi run rs-lint`. +`pixi run clean --runtime-only &&`. Before push run fmt and `pixi run rs-lint`. Proceed autonomously except when a design has multiple valid choices with no clear winner, or a test still fails after five root-cause-driven attempts. diff --git a/.githooks/pre-commit b/.githooks/pre-commit index 40f98579c..50a543345 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -1,7 +1,5 @@ #!/bin/bash -repo_root=$(git rev-parse --show-toplevel) - echo "Running pre-commit checks..." # Check Rust formatting @@ -18,7 +16,7 @@ if ! pixi run cargo fmt --all -- --check; then fi # Collect staged C++ files -staged_cpp=$(git diff --cached --name-only --diff-filter=ACM -- 'lib/crowdb-tree/src/*.cpp' 'lib/crowdb-tree/include/crowdb-tree/*.h') +staged_cpp=$(git diff --cached --name-only --diff-filter=ACM -- 'lib/crowdb-tree/*.cpp' 'lib/crowdb-tree/*.h') # Check C++ formatting (only staged files) if [ -n "$staged_cpp" ]; then @@ -39,63 +37,14 @@ if [ $? -ne 0 ]; then exit 1 fi -# Run clang-tidy (only on staged .cpp files) -staged_cpp_src=$(git diff --cached --name-only --diff-filter=ACM -- 'lib/crowdb-tree/src/*.cpp') -if [ -n "$staged_cpp_src" ] && [ -d lib/crowdb-tree/build ]; then - echo "Running clang-tidy on staged C++ files..." - # Mirror CMake's liburing gate: reactor.cpp, block_async_page_store.cpp, - # and any file that includes reactor.h require liburing headers; on macOS - # (and anywhere liburing is missing) they are excluded from the build, so - # clang-tidy must skip them too. clang-tidy uses clang (not GCC) and can't - # find liburing.h in the pixi env, so skip these files when liburing.h is - # not at a system path. Most crowdb-tree source files transitively include - # reactor.h via crowdb-tree.h → options.h, so when liburing.h is not at a - # system path, skip clang-tidy entirely for crowdb-tree sources. - have_liburing=false - if [ -f /opt/homebrew/include/liburing.h ] || [ -f /usr/local/include/liburing.h ] || [ -f /usr/include/liburing.h ]; then - have_liburing=true - fi - cpp_files=() - for f in $staged_cpp_src; do - case "$f" in - */reactor.cpp | */file_async_page_store.cpp | */block_async_page_store.cpp) - $have_liburing && cpp_files+=("$f") - ;; - *) - if $have_liburing; then - cpp_files+=("$f") - else - # Skip: clang-tidy can't find liburing.h (transitively - # included via crowdb-tree.h → options.h → reactor.h). - echo " Skipping clang-tidy for $f (liburing.h not at system path)" - fi - ;; - esac - done - if [ ${#cpp_files[@]} -gt 0 ]; then - tmpdir="$repo_root/.crowdb-runtime/artifacts/pre-commit/clang-tidy-$$" - mkdir -p "$tmpdir" - fail=0 - for f in "${cpp_files[@]}"; do - base=$(basename "$f") - (pixi run clang-tidy "$f" -p lib/crowdb-tree/build 2>&1 | grep -E '^(error:|warning:|/.*: error:|/.*: warning:)' > "$tmpdir/$base.out") & - done - wait - for f in "${cpp_files[@]}"; do - base=$(basename "$f") - if [ -s "$tmpdir/$base.out" ]; then - cat "$tmpdir/$base.out" - fail=1 - fi - done - rm -rf "$tmpdir" - if [ $fail -ne 0 ]; then - echo "❌ clang-tidy failed. Fix warnings before committing." - exit 1 - fi +# Use the same compile database, dependency include paths and changed-header +# scope as the repository C++ gate. Preserve warnings as commit failures. +if [ -n "$staged_cpp" ]; then + echo "Running tree-lint on staged C++ files..." + if ! CT_LINT_WARNINGS_AS_ERRORS=1 pixi run tree-lint $staged_cpp; then + echo "❌ tree-lint failed. Fix diagnostics before committing." + exit 1 fi -elif [ -n "$staged_cpp_src" ] && [ ! -d lib/crowdb-tree/build ]; then - echo "⚠️ Skipping clang-tidy: lib/crowdb-tree/build not found. Run 'pixi run build' first." fi echo "✅ All checks passed!" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4f5b9f89..180c1fcf4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,9 +62,9 @@ jobs: - name: Cache compilation uses: ./.github/actions/compilation-cache - name: Clean env - run: pixi run clean-env + run: pixi run clean --runtime-only - name: Build C++ libs - run: pixi run build-cpp + run: pixi run build - name: Run C++ and FFI tests run: pixi run test-cpp - name: Capture runner state on failure @@ -117,7 +117,7 @@ jobs: - name: Cache compilation uses: ./.github/actions/compilation-cache - name: Clean env - run: pixi run clean-env + run: pixi run clean --runtime-only - name: Run core component tests run: pixi run test-core - name: Capture runner state on failure @@ -170,9 +170,9 @@ jobs: - name: Cache compilation uses: ./.github/actions/compilation-cache - name: Clean env - run: pixi run clean-env + run: pixi run clean --runtime-only - name: Build C++ libs (crowdb-diskio subprocess) - run: pixi run build-cpp + run: pixi run build - name: Run storage component tests run: pixi run test-storage - name: Capture runner state on failure @@ -207,7 +207,7 @@ jobs: retention-days: 7 - name: Clean subprocesses if: always() - run: pixi run clean-env + run: pixi run clean --runtime-only AccessTests: timeout-minutes: 90 @@ -228,7 +228,7 @@ jobs: pixi-version: v0.77.0 - uses: ./.github/actions/compilation-cache - name: Clean env - run: pixi run clean-env + run: pixi run clean --runtime-only - name: Run access component tests run: | pixi run test-access @@ -272,7 +272,7 @@ jobs: retention-days: 7 - name: Clean subprocesses if: always() - run: pixi run clean-env + run: pixi run clean --runtime-only S3E2E: env: @@ -301,7 +301,7 @@ jobs: artifact: ${{ needs.BuildClients.outputs.artifact }} kind: clients - name: Clean env - run: pixi run clean-env + run: pixi run clean --runtime-only - uses: actions/download-artifact@v4 with: name: ${{ needs.ContainerImage.outputs.artifact }} @@ -335,7 +335,7 @@ jobs: retention-days: 7 - name: Clean subprocesses if: always() - run: pixi run clean-env + run: pixi run clean --runtime-only IcebergE2E: strategy: @@ -391,7 +391,7 @@ jobs: retention-days: 7 - name: Clean subprocesses if: always() - run: pixi run clean-env + run: pixi run clean --runtime-only ConsoleTests: runs-on: ubuntu-24.04 @@ -412,7 +412,7 @@ jobs: - name: Cache compilation uses: ./.github/actions/compilation-cache - name: Clean env - run: pixi run clean-env + run: pixi run clean --runtime-only - name: Run console tests run: pixi run test-console - name: Capture runner state on failure @@ -447,7 +447,7 @@ jobs: retention-days: 7 - name: Clean subprocesses if: always() - run: pixi run clean-env + run: pixi run clean --runtime-only UITests: runs-on: ubuntu-24.04 @@ -497,7 +497,7 @@ jobs: retention-days: 7 - name: Clean subprocesses if: always() - run: pixi run clean-env + run: pixi run clean --runtime-only BuildClients: runs-on: ubuntu-24.04 @@ -625,7 +625,7 @@ jobs: - name: Verify centralized KV fixtures and protocol layer run: | pixi run test-container-e2e-fixture - pixi run clean-env + pixi run clean --runtime-only pixi run test-e2e-kv - name: Record KV verified digest id: artifact diff --git a/.github/workflows/docker-preview.yml b/.github/workflows/docker-preview.yml index c6d9d1fdd..7924363dc 100644 --- a/.github/workflows/docker-preview.yml +++ b/.github/workflows/docker-preview.yml @@ -145,9 +145,9 @@ jobs: env: CROWDB_PREVIEW_TEST_ARTIFACTS: ${{ runner.temp }}/crowdb-preview-artifacts run: | - pixi run install-ui-deps + pixi run install-deps pixi run bash -c 'cd e2e-test/clients/console-ui && npm ci --ignore-scripts' - pixi run clean-env + pixi run clean --runtime-only pixi run --skip-deps test-single-node-container pixi run test-container-e2e --profile bridge-admission --concurrency 1 --node-cpus 1 --cpus 4 --memory-mib 6144 pixi run sudo --preserve-env=CROWDB_E2E_PRINT_SERVICE_ERRORS,CROWDB_COMPILED_TESTS "$(command -v pixi)" run --as-is test-container-e2e --profile host --concurrency 1 --cpus 4 --memory-mib 4096 @@ -247,7 +247,7 @@ jobs: - name: Verify centralized KV fixtures and protocol layer run: | pixi run test-container-e2e-fixture - pixi run clean-env + pixi run clean --runtime-only pixi run test-container-e2e --layer kv --concurrency 2 --require-isolation --cpus 4 --memory-mib 4096 - name: Record KV verified digest id: artifact diff --git a/.github/workflows/iceberg-container-ecosystem.yml b/.github/workflows/iceberg-container-ecosystem.yml index 6ab3ea7e7..11f09e6c9 100644 --- a/.github/workflows/iceberg-container-ecosystem.yml +++ b/.github/workflows/iceberg-container-ecosystem.yml @@ -125,4 +125,4 @@ jobs: retention-days: 7 - name: Clean subprocesses if: always() - run: pixi run clean-env + run: pixi run clean --runtime-only diff --git a/.github/workflows/release-container.yml b/.github/workflows/release-container.yml index 6d598f72e..889773fbb 100644 --- a/.github/workflows/release-container.yml +++ b/.github/workflows/release-container.yml @@ -147,9 +147,9 @@ jobs: env: CROWDB_PREVIEW_TEST_ARTIFACTS: ${{ runner.temp }}/crowdb-preview-artifacts run: | - pixi run install-ui-deps + pixi run install-deps pixi run bash -c 'cd e2e-test/clients/console-ui && npm ci --ignore-scripts' - pixi run clean-env + pixi run clean --runtime-only pixi run --skip-deps test-single-node-container pixi run test-container-e2e --profile bridge-admission --concurrency 1 --node-cpus 1 --cpus 4 --memory-mib 6144 pixi run sudo --preserve-env=CROWDB_E2E_PRINT_SERVICE_ERRORS,CROWDB_COMPILED_TESTS "$(command -v pixi)" run --as-is test-container-e2e --profile host --concurrency 1 --cpus 4 --memory-mib 4096 @@ -250,7 +250,7 @@ jobs: - name: Verify centralized KV fixtures and protocol layer run: | pixi run test-container-e2e-fixture - pixi run clean-env + pixi run clean --runtime-only pixi run test-container-e2e --layer kv --concurrency 2 --require-isolation --cpus 4 --memory-mib 4096 - name: Record KV verified digest id: artifact diff --git a/AGENTS.md b/AGENTS.md index c606ee69b..79396c51e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,6 +45,11 @@ and transport exposed through FFI. - Markdown is primarily read raw. Prefer bullets; use tables for real comparisons only. `doc/doc_index.md` always uses tables. When a table is used, pad columns with spaces so the raw markdown aligns visually. +- Console UI changes follow `doc/design/console/design-crowdb-console-ui.md`. + Before changing visible behavior, update the UI spec and its ASCII layout + after agreement with the user; only then implement the UI change. Do not + unilaterally change the spec or introduce UI behavior that is absent from + the agreed spec. Each visible UI change needs focused test or E2E coverage. ## Skill dispatch diff --git a/app/crowdb-access-server/src/config.rs b/app/crowdb-access-server/src/config.rs index 524b5901e..7607fb693 100644 --- a/app/crowdb-access-server/src/config.rs +++ b/app/crowdb-access-server/src/config.rs @@ -247,6 +247,8 @@ impl std::fmt::Debug for IcebergLocatorSigningConfig { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(default)] pub struct DatasetConfig { + /// Delay before reclaiming marked Datasets and Snapshots. + pub reclaim_grace_seconds: u64, pub listen: Option, pub max_in_flight: usize, pub max_batch_size: usize, @@ -256,6 +258,7 @@ pub struct DatasetConfig { impl Default for DatasetConfig { fn default() -> Self { Self { + reclaim_grace_seconds: 20 * 60, listen: None, max_in_flight: 8, max_batch_size: 10_000, @@ -425,6 +428,9 @@ impl BaseConfig for AccessConfig { impl AccessConfig { fn validate_dataset(&self) -> Result<(), String> { + if self.dataset.reclaim_grace_seconds == 0 { + return Err("dataset.reclaim_grace_seconds must be positive".into()); + } if self.dataset.max_in_flight == 0 || self.dataset.max_in_flight > 64 { return Err("dataset.max_in_flight must be between 1 and 64".into()); } diff --git a/app/crowdb-access-server/src/dataset.rs b/app/crowdb-access-server/src/dataset.rs index 76c1aeab1..1e11e221a 100644 --- a/app/crowdb-access-server/src/dataset.rs +++ b/app/crowdb-access-server/src/dataset.rs @@ -2,6 +2,8 @@ //! semantics are delegated to `crowdb-access-dataset`. mod api; +mod cleanup; +use cleanup::start_cleanup; mod legacy; mod models; @@ -11,7 +13,7 @@ use std::time::Duration; use crowdb_access_dataset::{ AuthorityError, ChunkKvDatasetStore, ChunkPayloadStorage, DatasetAuthority, DatasetReadService, - DatasetSnapshotRequest, DatasetUploads, UploadExpiryCursor, UploadedChunkReader, + DatasetSnapshotRequest, DatasetUploads, UploadedChunkReader, }; use crowdb_chunk_kv_client::{ ChunkKvClient, ChunkKvRpcTransport, ClientConfig as ChunkKvConfig, Group0ChunkKvRangeCatalogSource, @@ -35,6 +37,7 @@ fn body(bytes: Bytes) -> DatasetBody { Full::new(bytes).map_err(|never| match never {}).boxed_unsync() } +#[derive(Clone)] pub struct DatasetHttpService { reads: Arc, bearer: Option>, @@ -91,6 +94,8 @@ fn status_for_authority(error: &AuthorityError) -> StatusCode { match error { AuthorityError::Invalid(_) => StatusCode::BAD_REQUEST, AuthorityError::AlreadyExists + | AuthorityError::DatasetNotEmpty + | AuthorityError::DatasetReplaced | AuthorityError::ParentConflict | AuthorityError::HeadConflict | AuthorityError::SnapshotHeadConflict { .. } @@ -184,10 +189,10 @@ pub async fn run_with_shutdown( store.clone(), Arc::new(ChunkPayloadStorage::new(chunks.clone())), )); - let authority = Arc::new(DatasetAuthority::with_chunk_reader( - store, - Arc::new(UploadedChunkReader::new((*uploads).clone())), - )); + let authority = Arc::new( + DatasetAuthority::with_chunk_reader(store, Arc::new(UploadedChunkReader::new((*uploads).clone()))) + .with_reclaim_grace_seconds(access.dataset.reclaim_grace_seconds), + ); let identity = crowdb_access_dataset::DatasetIdentity::new( crowdb_access_dataset::NamespacePath::root(), std::env::var("CROWDB_DATASET_NAME").unwrap_or_else(|_| "default".into()), @@ -223,102 +228,6 @@ pub async fn run_with_shutdown( Ok(()) } -fn start_cleanup( - authority: Arc, - uploads: Option>, -) -> tokio::task::JoinHandle<()> { - tokio::spawn(async move { - let mut snapshot_cursor = crowdb_access_dataset::SnapshotReclaimSweep::default(); - let mut next_snapshot = tokio::time::Instant::now(); - let mut candidate_cursor = crowdb_access_dataset::SnapshotExpirySweep::default(); - let mut next_candidate = tokio::time::Instant::now(); - let mut composition_cursor = crowdb_access_dataset::composition::CompositionReclaimSweep::default(); - let mut next_composition = tokio::time::Instant::now(); - let mut split_cursor = crowdb_access_dataset::split::SplitReclaimSweep::default(); - let mut next_split = tokio::time::Instant::now(); - let mut revision_cursor = crowdb_access_dataset::split::SplitRevisionReclaimSweep::default(); - let mut next_revision = tokio::time::Instant::now(); - let mut upload_cursor = UploadExpiryCursor::default(); - let mut next_upload = tokio::time::Instant::now(); - loop { - tokio::select! { - () = tokio::time::sleep_until(next_revision) => { - let result = authority.reclaim_split_revisions_page(&mut revision_cursor, - &crowdb_access_dataset::ReadCancellation::new()).await; - let delay = if revision_cursor.has_more() { - if result.is_ok() { Duration::from_millis(50) } else { Duration::from_millis(500) } - } else { Duration::from_secs(15) }; - next_revision = tokio::time::Instant::now() + delay; - if let Err(error) = result { - tracing::error!(%error, "Dataset Split revision cleanup failed; deletion marks retained for retry"); - } - } - () = tokio::time::sleep_until(next_candidate) => { - let result = authority.expire_snapshot_candidates_page(&mut candidate_cursor, - &crowdb_access_dataset::ReadCancellation::new()).await; - let delay = if candidate_cursor.has_more() { - if result.is_ok() { Duration::from_millis(50) } else { Duration::from_millis(500) } - } else { Duration::from_secs(15) }; - next_candidate = tokio::time::Instant::now() + delay; - if let Err(error) = result { - tracing::error!(%error, "Dataset candidate expiry failed; resource deadlines retained for retry"); - } - } - () = tokio::time::sleep_until(next_split) => { - let result = authority.reclaim_splits_page(&mut split_cursor, - &crowdb_access_dataset::ReadCancellation::new()).await; - let delay = if split_cursor.has_more() { - if result.is_ok() { Duration::from_millis(50) } else { Duration::from_millis(500) } - } else { Duration::from_secs(15) }; - next_split = tokio::time::Instant::now() + delay; - if let Err(error) = result { - tracing::error!(%error, "Dataset legacy Split cleanup failed; resource facts retained for retry"); - } - } - () = tokio::time::sleep_until(next_composition), if uploads.is_some() => { - let Some(uploads) = &uploads else { continue }; - let result = authority.reclaim_compositions_page(uploads, &mut composition_cursor, - &crowdb_access_dataset::ReadCancellation::new()).await; - let delay = if composition_cursor.has_more() { - if result.is_ok() { Duration::from_millis(50) } else { Duration::from_millis(500) } - } else { Duration::from_secs(15) }; - next_composition = tokio::time::Instant::now() + delay; - if let Err(error) = result { - tracing::error!(%error, "Dataset legacy Composition cleanup failed; resource facts retained for retry"); - } - } - () = tokio::time::sleep_until(next_snapshot), if uploads.is_some() => { - let Some(uploads) = &uploads else { continue }; - let result = authority.reclaim_due_page(uploads, &mut snapshot_cursor, - &crowdb_access_dataset::ReadCancellation::new()).await; - let delay = if snapshot_cursor.has_more() { - if result.is_ok() { Duration::from_millis(50) } else { Duration::from_millis(500) } - } else { Duration::from_secs(15) }; - next_snapshot = tokio::time::Instant::now() + delay; - if let Err(error) = result { - if matches!(error, crowdb_access_dataset::SnapshotReclaimError::Limit) { - tracing::debug!("Dataset snapshot cleanup yielded at its work deadline"); - } else { - tracing::error!(%error, "Dataset snapshot cleanup failed; durable deletion marks retained for retry"); - } - } - } - () = tokio::time::sleep_until(next_upload), if uploads.is_some() => { - let Some(uploads) = &uploads else { continue }; - let result = uploads.expire_step(unix_seconds(), &mut upload_cursor).await; - let delay = if upload_cursor.has_more() { - if result.is_ok() { Duration::from_millis(50) } else { Duration::from_millis(500) } - } else { Duration::from_secs(15) }; - next_upload = tokio::time::Instant::now() + delay; - if let Err(error) = result { - tracing::error!(%error, "Dataset upload cleanup failed; retaining cleanup records for retry"); - } - } - } - } - }) -} - /// Starts the real bounded cleanup loop without opening a listener. #[cfg(feature = "test-util")] #[must_use] diff --git a/app/crowdb-access-server/src/dataset/api.rs b/app/crowdb-access-server/src/dataset/api.rs index 84e8335e6..26e8b86e5 100644 --- a/app/crowdb-access-server/src/dataset/api.rs +++ b/app/crowdb-access-server/src/dataset/api.rs @@ -58,7 +58,9 @@ impl DatasetHttpService { self.request_timeout.max(std::time::Duration::from_secs(1800)) } else if request.uri().path().starts_with("/api/dataset/v2/") && (matches!(*request.method(), Method::GET | Method::HEAD) - || (*request.method() == Method::POST && request.uri().path().ends_with("/matrix"))) + || (*request.method() == Method::POST + && (request.uri().path().ends_with("/matrix") + || request.uri().path().ends_with("/split-preview")))) { self.request_timeout.min(std::time::Duration::from_secs(5)) } else { diff --git a/app/crowdb-access-server/src/dataset/api/error.rs b/app/crowdb-access-server/src/dataset/api/error.rs index 8c721fc70..5614ac348 100644 --- a/app/crowdb-access-server/src/dataset/api/error.rs +++ b/app/crowdb-access-server/src/dataset/api/error.rs @@ -96,6 +96,12 @@ impl From for ApiError { } impl From for ApiError { fn from(error: AuthorityError) -> Self { + if matches!(error, AuthorityError::DatasetNotEmpty) { + return Self::new(StatusCode::CONFLICT, "dataset_not_empty"); + } + if matches!(error, AuthorityError::DatasetReplaced) { + return Self::new(StatusCode::CONFLICT, "dataset_replaced"); + } if let AuthorityError::SnapshotHeadConflict { observed } = error { return Self::conflict().with_observed_head(serde_json::json!({ "latest":observed.latest.map(|id|id.to_string()), diff --git a/app/crowdb-access-server/src/dataset/api/upload.rs b/app/crowdb-access-server/src/dataset/api/upload.rs index 80f212966..9e6aa94d9 100644 --- a/app/crowdb-access-server/src/dataset/api/upload.rs +++ b/app/crowdb-access-server/src/dataset/api/upload.rs @@ -43,8 +43,13 @@ impl DatasetHttpService { match (request.method().clone(), tail) { (Method::POST, ["uploads"]) => { let input: UploadRequest = read_json(request).await?; - let location = DatasetUploads::location(identity, operation(&input.id)?)?; let md5 = digest(&input.md5)?; + DatasetUploads::validate_creation(input.length, input.part_size)?; + let location = DatasetUploads::location(identity, operation(&input.id)?)?; + self.reads + .authority() + .reserve_upload_creation(identity, operation(&input.id)?) + .await?; let record = uploads .create( &location, diff --git a/app/crowdb-access-server/src/dataset/api/v2.rs b/app/crowdb-access-server/src/dataset/api/v2.rs index 1f9014cfe..60881b1d5 100644 --- a/app/crowdb-access-server/src/dataset/api/v2.rs +++ b/app/crowdb-access-server/src/dataset/api/v2.rs @@ -18,6 +18,7 @@ mod read_models; mod reads; mod split_models; mod split_preparation; +mod split_preview; mod split_requests; mod split_responses; mod splits; @@ -36,6 +37,35 @@ struct Capabilities { impl DatasetHttpService { pub(super) async fn route_v2(&self, parts: &[String], request: Request) -> ApiResult { + const HEADER: &str = "x-crowdb-dataset-id"; + if request.headers().get_all(HEADER).iter().count() > 1 { + return Err(ApiError::invalid()); + } + if let Some(value) = request.headers().get(HEADER) { + if parts.len() < 7 + || parts[..6] != ["api", "dataset", "v2", "namespaces", "dataset-ns", "datasets"] + { + return Err(ApiError::invalid()); + } + let value = value.to_str().map_err(|_| ApiError::invalid())?; + let bytes = hex::decode(value).map_err(|_| ApiError::invalid())?; + let id = crowdb_access_dataset::DatasetId::from_bytes(&bytes).map_err(|_| ApiError::invalid())?; + if id.to_string() != value { + return Err(ApiError::invalid()); + } + let scoped = Self { + reads: std::sync::Arc::new(self.reads.for_dataset_instance(id)), + ..self.clone() + }; + let identity = + DatasetIdentity::new(NamespacePath::root(), &parts[6]).map_err(|_| ApiError::invalid())?; + scoped.reads.authority().get_dataset(&identity).await?; + return scoped.route_v2_scoped(parts, request).await; + } + self.route_v2_scoped(parts, request).await + } + + async fn route_v2_scoped(&self, parts: &[String], request: Request) -> ApiResult { if parts == ["api", "dataset", "v2", "namespaces", "dataset-ns", "datasets"] { return self.v2_catalog(request).await; } @@ -48,7 +78,10 @@ impl DatasetHttpService { "bounded-inspection", "inspection-preparation", "dataset-management", + "dataset-delete", + "dataset-identity-fence", "split-revisions", + "split-preview", "split-revision-delete", "split-preparation", "split-head-update", @@ -76,7 +109,7 @@ impl DatasetHttpService { if matches!( tail.as_slice(), [] | ["names", "stable"] - | ["commits", _] + | ["deletion" | "commits", _] | ["snapshots", _, "retain" | "release" | "reclaim"] | ["split-drafts", _, "cleanup"] ) { @@ -92,6 +125,11 @@ impl DatasetHttpService { .await; } } + if let ["snapshots", id, "split-preview"] = tail.as_slice() { + return self + .v2_split_preview(&identity, super::snapshot(id)?, request) + .await; + } if let ["snapshots", id] = tail.as_slice() { if request.method() == Method::PUT { return self.v2_prepare_snapshot(&identity, id, request).await; diff --git a/app/crowdb-access-server/src/dataset/api/v2/auxiliary.rs b/app/crowdb-access-server/src/dataset/api/v2/auxiliary.rs index dcfa8fa71..cb902e4a1 100644 --- a/app/crowdb-access-server/src/dataset/api/v2/auxiliary.rs +++ b/app/crowdb-access-server/src/dataset/api/v2/auxiliary.rs @@ -14,6 +14,18 @@ use serde::Deserialize; struct Stable { snapshot: String, } +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct DeleteDataset { + id: String, +} +fn dataset_id(value: &str) -> Result { + crowdb_access_dataset::DatasetId::from_bytes(operation(value)?.as_bytes()) + .map_err(|_| ApiError::invalid()) +} +fn deletion(value: &crowdb_access_dataset::DatasetDeletion) -> serde_json::Value { + serde_json::json!({"id":value.id.to_string(), "reclaim_after":value.reclaim_after.to_string(), "complete":value.complete}) +} impl DatasetHttpService { pub(super) async fn v2_auxiliary( &self, @@ -24,6 +36,19 @@ impl DatasetHttpService { Query::parse(request.uri().query(), &[])?; let authority = self.reads.authority(); match (request.method().clone(), tail) { + (Method::DELETE, []) => { + let input: DeleteDataset = read_json(request).await?; + let result = authority + .delete_dataset(identity, dataset_id(&input.id)?, &ReadCancellation::new()) + .await?; + return json(&deletion(&result)); + } + (Method::GET, ["deletion", id]) => { + read_body(request, false).await?; + return json( + &serde_json::json!({"deletion":authority.dataset_deletion(identity, dataset_id(id)?).await?.as_ref().map(deletion)}), + ); + } (Method::PUT, []) => { read_body(request, false).await?; authority.create_dataset(identity.clone()).await?; diff --git a/app/crowdb-access-server/src/dataset/api/v2/split_preview.rs b/app/crowdb-access-server/src/dataset/api/v2/split_preview.rs new file mode 100644 index 000000000..8040c5315 --- /dev/null +++ b/app/crowdb-access-server/src/dataset/api/v2/split_preview.rs @@ -0,0 +1,36 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::super::{json, read_json, ApiError, ApiResult, DatasetHttpService}; +use super::{inspection_models::Page, query::Query, split_models::DraftSplit}; +use base64::{engine::general_purpose::STANDARD, Engine}; +use crowdb_access_dataset::{DatasetIdentity, ReadCancellation, SnapshotId}; +use hyper::{body::Incoming, Method, Request}; + +impl DatasetHttpService { + pub(super) async fn v2_split_preview( + &self, + identity: &DatasetIdentity, + snapshot: SnapshotId, + request: Request, + ) -> ApiResult { + if request.method() != Method::POST { + return Err(ApiError::not_found()); + } + let query = Query::parse(request.uri().query(), &["after", "limit"])?; + let input: DraftSplit = read_json(request).await?; + let page = self + .reads + .authority() + .inspection_split_page( + identity, + snapshot, + &input.native()?, + query.after()?, + query.budget()?, + &ReadCancellation::new(), + ) + .await?; + json(&Page::map(page, |id| STANDARD.encode(id))) + } +} diff --git a/app/crowdb-access-server/src/dataset/api/v2/uploads.rs b/app/crowdb-access-server/src/dataset/api/v2/uploads.rs index b75af7594..19d88cc56 100644 --- a/app/crowdb-access-server/src/dataset/api/v2/uploads.rs +++ b/app/crowdb-access-server/src/dataset/api/v2/uploads.rs @@ -31,15 +31,16 @@ impl DatasetHttpService { let now = crate::dataset::unix_seconds(); if tail == ["uploads"] && request.method() == Method::POST { let input: Create = read_json(request).await?; + let length = decimal(&input.length)?; + let md5 = digest(&input.md5)?; + DatasetUploads::validate_creation(length, input.part_size)?; let location = DatasetUploads::location(identity, operation(&input.id)?)?; + self.reads + .authority() + .reserve_upload_creation(identity, operation(&input.id)?) + .await?; let record = uploads - .create( - &location, - decimal(&input.length)?, - input.part_size, - digest(&input.md5)?, - now.saturating_add(86400), - ) + .create(&location, length, input.part_size, md5, now.saturating_add(86400)) .await?; return json(&wire(&input.id, record)); } diff --git a/app/crowdb-access-server/src/dataset/cleanup.rs b/app/crowdb-access-server/src/dataset/cleanup.rs new file mode 100644 index 000000000..9e59cfb33 --- /dev/null +++ b/app/crowdb-access-server/src/dataset/cleanup.rs @@ -0,0 +1,110 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. +use super::{unix_seconds, DatasetAuthority, DatasetUploads}; +use crowdb_access_dataset::UploadExpiryCursor; +use std::{sync::Arc, time::Duration}; + +fn next_delay(more: bool, success: bool, idle: Duration) -> Duration { + if more { + Duration::from_millis(if success { 50 } else { 500 }) + } else { + idle + } +} + +pub(super) fn start_cleanup( + authority: Arc, + uploads: Option>, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + let reclaim_poll = Duration::from_secs(authority.reclaim_grace_seconds().clamp(1, 15)); + let mut dataset_cursor = crowdb_access_dataset::DatasetDeletionSweep::default(); + let mut next_dataset = tokio::time::Instant::now(); + let mut snapshot_cursor = crowdb_access_dataset::SnapshotReclaimSweep::default(); + let mut next_snapshot = tokio::time::Instant::now(); + let mut candidate_cursor = crowdb_access_dataset::SnapshotExpirySweep::default(); + let mut next_candidate = tokio::time::Instant::now(); + let mut composition_cursor = crowdb_access_dataset::composition::CompositionReclaimSweep::default(); + let mut next_composition = tokio::time::Instant::now(); + let mut split_cursor = crowdb_access_dataset::split::SplitReclaimSweep::default(); + let mut next_split = tokio::time::Instant::now(); + let mut revision_cursor = crowdb_access_dataset::split::SplitRevisionReclaimSweep::default(); + let mut next_revision = tokio::time::Instant::now(); + let mut upload_cursor = UploadExpiryCursor::default(); + let mut next_upload = tokio::time::Instant::now(); + loop { + tokio::select! { + () = tokio::time::sleep_until(next_dataset) => { + let result = authority.reclaim_datasets_page(&mut dataset_cursor, + &crowdb_access_dataset::ReadCancellation::new()).await; + let delay = next_delay(dataset_cursor.has_more(), result.is_ok(), reclaim_poll); + next_dataset = tokio::time::Instant::now() + delay; + if let Err(error) = result { + tracing::error!(%error, "Dataset namespace cleanup failed; durable deletion marks retained for retry"); + } + } + + () = tokio::time::sleep_until(next_revision) => { + let result = authority.reclaim_split_revisions_page(&mut revision_cursor, + &crowdb_access_dataset::ReadCancellation::new()).await; + let delay = next_delay(revision_cursor.has_more(), result.is_ok(), reclaim_poll); + next_revision = tokio::time::Instant::now() + delay; + if let Err(error) = result { + tracing::error!(%error, "Dataset Split revision cleanup failed; deletion marks retained for retry"); + } + } + () = tokio::time::sleep_until(next_candidate) => { + let result = authority.expire_snapshot_candidates_page(&mut candidate_cursor, + &crowdb_access_dataset::ReadCancellation::new()).await; + let delay = next_delay(candidate_cursor.has_more(), result.is_ok(), Duration::from_secs(15)); + next_candidate = tokio::time::Instant::now() + delay; + if let Err(error) = result { + tracing::error!(%error, "Dataset candidate expiry failed; resource deadlines retained for retry"); + } + } + () = tokio::time::sleep_until(next_split) => { + let result = authority.reclaim_splits_page(&mut split_cursor, + &crowdb_access_dataset::ReadCancellation::new()).await; + let delay = next_delay(split_cursor.has_more(), result.is_ok(), Duration::from_secs(15)); + next_split = tokio::time::Instant::now() + delay; + if let Err(error) = result { + tracing::error!(%error, "Dataset legacy Split cleanup failed; resource facts retained for retry"); + } + } + () = tokio::time::sleep_until(next_composition), if uploads.is_some() => { + let Some(uploads) = &uploads else { continue }; + let result = authority.reclaim_compositions_page(uploads, &mut composition_cursor, + &crowdb_access_dataset::ReadCancellation::new()).await; + let delay = next_delay(composition_cursor.has_more(), result.is_ok(), Duration::from_secs(15)); + next_composition = tokio::time::Instant::now() + delay; + if let Err(error) = result { + tracing::error!(%error, "Dataset legacy Composition cleanup failed; resource facts retained for retry"); + } + } + () = tokio::time::sleep_until(next_snapshot), if uploads.is_some() => { + let Some(uploads) = &uploads else { continue }; + let result = authority.reclaim_due_page(uploads, &mut snapshot_cursor, + &crowdb_access_dataset::ReadCancellation::new()).await; + let delay = next_delay(snapshot_cursor.has_more(), result.is_ok(), reclaim_poll); + next_snapshot = tokio::time::Instant::now() + delay; + if let Err(error) = result { + if matches!(error, crowdb_access_dataset::SnapshotReclaimError::Limit) { + tracing::debug!("Dataset snapshot cleanup yielded at its work deadline"); + } else { + tracing::error!(%error, "Dataset snapshot cleanup failed; durable deletion marks retained for retry"); + } + } + } + () = tokio::time::sleep_until(next_upload), if uploads.is_some() => { + let Some(uploads) = &uploads else { continue }; + let result = uploads.expire_step(unix_seconds(), &mut upload_cursor).await; + let delay = next_delay(upload_cursor.has_more(), result.is_ok(), Duration::from_secs(15)); + next_upload = tokio::time::Instant::now() + delay; + if let Err(error) = result { + tracing::error!(%error, "Dataset upload cleanup failed; retaining cleanup records for retry"); + } + } + } + } + }) +} diff --git a/app/crowdb-access-server/src/iceberg/http.rs b/app/crowdb-access-server/src/iceberg/http.rs index f58ecfb79..a77bf32d1 100644 --- a/app/crowdb-access-server/src/iceberg/http.rs +++ b/app/crowdb-access-server/src/iceberg/http.rs @@ -387,6 +387,11 @@ impl IcebergHttpService { let warehouse = warehouse(query)?; let mut config = CatalogConfig::for_capabilities(warehouse.as_deref(), capabilities)?; config.endpoints = Route::endpoints(&self.installed_routes(), capabilities); + if self.table_writes.is_some() { + config + .overrides + .insert("crowdb.guarded-maintenance".into(), "true".into()); + } Ok(response( 200, serde_json::to_vec(&config).map_err(|_| service_unavailable())?, diff --git a/app/crowdb-access-server/src/iceberg/inspection/pages.rs b/app/crowdb-access-server/src/iceberg/inspection/pages.rs index b315918ad..ae4a51038 100644 --- a/app/crowdb-access-server/src/iceberg/inspection/pages.rs +++ b/app/crowdb-access-server/src/iceberg/inspection/pages.rs @@ -27,6 +27,7 @@ impl Selection<'_> { ) .await .map_err(failed)?; + let rewritable = present::rewritable_schema(reader.metadata(), false); let mut cursor = self.cursor(&record)?; if cursor.block > 0 { reader @@ -41,7 +42,7 @@ impl Selection<'_> { let block_start = reader.position(); let Some(block) = reader.next().await.map_err(failed)? else { return Ok( - json!({"kind":"manifest-list","location":record.location.to_string(),"size":record.length,"rows":rows,"next":null,"offset":start}), + json!({"kind":"manifest-list","location":record.location.to_string(),"size":record.length,"rows":rows,"rewrite_safe":rewritable,"next":null,"offset":start}), ); }; bytes += block.payload.length; @@ -84,7 +85,7 @@ impl Selection<'_> { }, )?; return Ok( - json!({"kind":"manifest-list","location":record.location.to_string(),"size":record.length,"rows":rows,"next":next,"offset":start}), + json!({"kind":"manifest-list","location":record.location.to_string(),"size":record.length,"rows":rows,"rewrite_safe":rewritable,"next":next,"offset":start}), ); } size += cost; @@ -117,6 +118,7 @@ impl Selection<'_> { let writer = ManifestMetadata::parse(reader.metadata()).map_err(failed)?; let version = writer.version; let schema: Value = serde_json::from_slice(writer.schema_json).map_err(failed)?; + let rewritable = present::rewritable_schema(reader.metadata(), true); let mut cursor = self.cursor(&record)?; let start = cursor.index; let mut state = @@ -143,7 +145,7 @@ impl Selection<'_> { let row_id = state.next_row_id(); let Some(block) = reader.next().await.map_err(failed)? else { return Ok( - json!({"kind":"manifest","location":record.location.to_string(),"descriptor":present::manifest(list),"codec":codec,"schema":schema,"rows":rows,"next":null,"offset":start}), + json!({"kind":"manifest","location":record.location.to_string(),"descriptor":present::manifest(list),"codec":codec,"schema":schema,"rows":rows,"rewrite_safe":rewritable,"next":null,"offset":start}), ); }; bytes += block.payload.length; @@ -189,7 +191,7 @@ impl Selection<'_> { }, )?; return Ok( - json!({"kind":"manifest","location":record.location.to_string(),"descriptor":present::manifest(list),"codec":codec,"schema":schema,"rows":rows,"next":next,"offset":start}), + json!({"kind":"manifest","location":record.location.to_string(),"descriptor":present::manifest(list),"codec":codec,"schema":schema,"rows":rows,"rewrite_safe":rewritable,"next":next,"offset":start}), ); } size += cost; diff --git a/app/crowdb-access-server/src/iceberg/inspection/present.rs b/app/crowdb-access-server/src/iceberg/inspection/present.rs index d396533be..87805f1aa 100644 --- a/app/crowdb-access-server/src/iceberg/inspection/present.rs +++ b/app/crowdb-access-server/src/iceberg/inspection/present.rs @@ -3,6 +3,52 @@ use crowdb_access_iceberg::manifest::{ }; use serde_json::{json, Value}; +// Only the bounded console writer's scalar layouts can be reconstructed from +// inspection rows. Reject extra fields (including encryption metadata) rather +// than silently losing them during a manifest rewrite. +pub(super) fn rewritable_schema( + metadata: &std::collections::BTreeMap>, + manifest: bool, +) -> bool { + let Some(bytes) = metadata.get("avro.schema") else { + return false; + }; + let Ok(schema) = serde_json::from_slice::(bytes) else { + return false; + }; + let fields_match = |schema: &Value, expected: &[i64]| { + schema["fields"].as_array().is_some_and(|fields| { + fields.len() == expected.len() + && expected + .iter() + .all(|id| fields.iter().any(|field| field["field-id"].as_i64() == Some(*id))) + }) + }; + if !manifest { + return fields_match( + &schema, + &[500, 501, 502, 503, 517, 515, 516, 504, 505, 506, 512, 513, 514], + ); + } + if !fields_match(&schema, &[0, 1, 2, 3, 4]) { + return false; + } + let Some(data) = schema["fields"] + .as_array() + .and_then(|fields| fields.iter().find(|field| field["field-id"] == 2)) + else { + return false; + }; + if !fields_match(&data["type"], &[134, 100, 101, 102, 103, 104]) { + return false; + } + data["type"]["fields"] + .as_array() + .and_then(|fields| fields.iter().find(|field| field["field-id"] == 102)) + .and_then(|field| field["type"]["fields"].as_array()) + .is_some_and(Vec::is_empty) +} + pub(super) fn stringify_integers(value: &mut Value) { match value { Value::Number(number) if number.is_i64() || number.is_u64() => { diff --git a/app/crowdb-access-server/src/iceberg/table_write/mutation.rs b/app/crowdb-access-server/src/iceberg/table_write/mutation.rs index db1dd2d97..d8ef79fec 100644 --- a/app/crowdb-access-server/src/iceberg/table_write/mutation.rs +++ b/app/crowdb-access-server/src/iceberg/table_write/mutation.rs @@ -148,6 +148,11 @@ impl TableWrites { fn direct_error(error: DirectCommitError) -> IcebergErrorResponse { match error { + DirectCommitError::ReferencedSnapshot => IcebergErrorResponse::new( + 409, + "CommitFailedException", + "Cannot expire a current or referenced snapshot", + ), DirectCommitError::Conflict => { IcebergErrorResponse::new(409, "CommitFailedException", "Table generation changed") } diff --git a/app/crowdb-access-server/src/s3/operations.rs b/app/crowdb-access-server/src/s3/operations.rs index 284b5004c..bb1eebc34 100644 --- a/app/crowdb-access-server/src/s3/operations.rs +++ b/app/crowdb-access-server/src/s3/operations.rs @@ -7,6 +7,7 @@ use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; use base64::Engine as _; +use crowdb_access_s3::bucket::{BucketListRequest, BucketListSigner}; use crowdb_access_s3::condition::ObjectConditions; use crowdb_access_s3::continuation::ContinuationTokenSigner; use crowdb_access_s3::error::{S3Error, S3ErrorCode}; @@ -95,6 +96,7 @@ pub struct ProductionS3Operations { pub(super) storage: S3StorageClients, pub(super) config: S3ServiceConfig, signer: ContinuationTokenSigner, + bucket_signer: BucketListSigner, execution: S3Executor, metrics: Option>, } @@ -108,11 +110,14 @@ impl ProductionS3Operations { pub fn new(storage: S3StorageClients, config: S3ServiceConfig) -> Result { let signer = ContinuationTokenSigner::new(config.continuation_key.clone()) .map_err(|_| S3ErrorCode::InvalidRequest)?; + let bucket_signer = BucketListSigner::new(config.continuation_key.clone()) + .map_err(|_| S3ErrorCode::InvalidRequest)?; Ok(Self { execution: S3Executor::new(storage.clone(), config.tenant.clone()), storage, config, signer, + bucket_signer, metrics: None, }) } @@ -136,7 +141,7 @@ impl ProductionS3Operations { let result = match operation { S3Operation::CreateBucket => self.create_bucket(route).await, S3Operation::HeadBucket => self.head_bucket(route).await, - S3Operation::ListBuckets => self.list_buckets().await, + S3Operation::ListBuckets => self.list_buckets(&request).await, S3Operation::DeleteBucket => self.delete_bucket(route).await, S3Operation::PutObject => self.put_object(route, request).await, S3Operation::HeadObject => self.head_object(route, &request).await, @@ -171,9 +176,34 @@ impl ProductionS3Operations { response(StatusCode::OK, Vec::new()) } - async fn list_buckets(&self) -> Result, S3ErrorCode> { - let buckets = self.execution.list_buckets(DEFAULT_LIST_LIMIT).await?; - xml_response(wire::list_buckets(self.config.tenant.as_bytes(), &buckets)) + async fn list_buckets(&self, request: &Request) -> Result, S3ErrorCode> { + let query = Query::new(request.uri().query()); + let prefix = query.bytes("prefix").unwrap_or_default(); + let continuation = query.text("continuation-token"); + let max_buckets = query + .text("max-buckets") + .map_or(Ok(DEFAULT_LIST_LIMIT), |value| value.parse::()) + .map_err(|_| S3ErrorCode::InvalidRequest)?; + let page = self + .execution + .list_bucket_page( + &self.bucket_signer, + &BucketListRequest { + prefix: &prefix, + continuation: continuation.as_deref(), + max_buckets, + max_scan_items: self.config.list_scan_items, + max_scan_bytes: self.config.list_scan_bytes, + now: unix_seconds(), + ttl: self.config.continuation_ttl_seconds, + }, + ) + .await?; + xml_response(wire::list_bucket_page( + self.config.tenant.as_bytes(), + &prefix, + &page, + )) } async fn delete_bucket(&self, route: S3Route) -> Result, S3ErrorCode> { diff --git a/app/crowdb-access-server/tests/README.md b/app/crowdb-access-server/tests/README.md index 72d334bff..c240a481f 100644 --- a/app/crowdb-access-server/tests/README.md +++ b/app/crowdb-access-server/tests/README.md @@ -874,7 +874,7 @@ pixi run taskset -c 2 cargo test --release --locked -p crowdb-access-dataset --t ## Verification and execution notes - Library: `pixi run -- cargo test -p crowdb-access-iceberg --all-targets`. -- HTTP: `pixi run clean-env && pixi run -- cargo test -p crowdb-access-server --all-targets`. +- HTTP: `pixi run clean --runtime-only && pixi run -- cargo test -p crowdb-access-server --all-targets`. Default server tests alone skip the Iceberg suites. - SDK: `pixi run -- cargo test -p crowdb-access-server --features iceberg-e2e --test iceberg_table_sdk_test -- --ignored --nocapture --test-threads=1`. - Namespace SDK: `pixi run -- cargo test -p crowdb-access-server --features iceberg-e2e --test iceberg_namespace_sdk_test -- --ignored --nocapture`. @@ -890,7 +890,7 @@ pixi run taskset -c 2 cargo test --release --locked -p crowdb-access-dataset --t - For Java tests, use default Pixi for Cargo; set `JAVA_HOME=$PWD/.pixi/envs/iceberg-e2e/lib/jvm` and `CROWDB_ICEBERG_E2E_MVN=$PWD/.pixi/envs/iceberg-e2e/bin/mvn`. -- Prefix native tests with clean-env using the same isolated +- Prefix native tests with `clean --runtime-only` using the same isolated `CROWDB_RUNTIME_ROOT=$PWD/.crowdb-runtime/ephemeral/iceberg-catalog-e2e`; preserve unrelated persistent port claims. Do not clean while another test runs. - Gates: `pixi run -- cargo fmt --all -- --check`, `pixi run rs-lint`, and diff --git a/app/crowdb-access-server/tests/config_test.rs b/app/crowdb-access-server/tests/config_test.rs index 5cd0a826b..4c7d5558c 100644 --- a/app/crowdb-access-server/tests/config_test.rs +++ b/app/crowdb-access-server/tests/config_test.rs @@ -218,3 +218,13 @@ fn local_one_and_three_node_mirrors_are_explicit_and_never_relax_production() { ); } } + +#[test] +fn dataset_reclaim_grace_is_configurable() { + let mut config = crowdb_access_server::config::AccessConfig::default(); + assert_eq!(config.dataset.reclaim_grace_seconds, 1200); + config.dataset.reclaim_grace_seconds = 1; + config.validate().unwrap(); + config.dataset.reclaim_grace_seconds = 0; + assert!(config.validate().unwrap_err().contains("reclaim_grace_seconds")); +} diff --git a/app/crowdb-access-server/tests/dataset_composition_preparation_test.rs b/app/crowdb-access-server/tests/dataset_composition_preparation_test.rs index 7f5311cd4..21f1e2e20 100644 --- a/app/crowdb-access-server/tests/dataset_composition_preparation_test.rs +++ b/app/crowdb-access-server/tests/dataset_composition_preparation_test.rs @@ -41,7 +41,8 @@ async fn complete_resource_reuses_the_snapshot_id_across_restart_without_advanci let before = counts(&store); let prepared = response(client.put(&url).json(&input()), 200).await; let after = counts(&store); - assert_eq!((after.0 - before.0, after.1 - before.1), (12, 16)); + // Creation reserves the Dataset root: two admission reads and one root CAS. + assert_eq!((after.0 - before.0, after.1 - before.1), (14, 17)); assert_eq!(prepared["snapshot"], id); assert_eq!(prepared["sample_count"], "1"); assert!(prepared["initial_split"].is_string()); diff --git a/app/crowdb-access-server/tests/dataset_deletion_contract_test.rs b/app/crowdb-access-server/tests/dataset_deletion_contract_test.rs new file mode 100644 index 000000000..fb3715fdc --- /dev/null +++ b/app/crowdb-access-server/tests/dataset_deletion_contract_test.rs @@ -0,0 +1,126 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. +#[path = "common/dataset.rs"] +mod dataset; +use crowdb_access_dataset::*; +use dataset::{TestDatasetHttp, TestStore}; +use reqwest::{Client, StatusCode}; +use serde_json::{json, Value}; +use std::sync::Arc; + +#[tokio::test] +async fn http_delete_hides_dataset_and_retries_preserve_receipt() { + let service = TestDatasetHttp::start_payload(Arc::new(TestStore::default())).await; + let client = Client::new(); + let url = format!( + "{}/api/dataset/v2/namespaces/dataset-ns/datasets/empty", + service.origin + ); + let created: Value = client + .put(&url) + .bearer_auth("secret") + .send() + .await + .unwrap() + .error_for_status() + .unwrap() + .json() + .await + .unwrap(); + let id = created["id"].as_str().unwrap(); + // Rejected input must not leave a durable reservation that makes an empty Dataset nonempty. + for (length, part_size, md5) in [ + ("0", 4, "1".repeat(32)), + ("4", 0, "1".repeat(32)), + ("4", 4, "bad".into()), + ] { + let invalid = client.post(format!("{url}/uploads")).bearer_auth("secret") + .json(&json!({"id": OperationId::random().to_string(), "length":length, "part_size":part_size, "md5":md5})) + .send().await.unwrap(); + assert_eq!(invalid.status(), StatusCode::BAD_REQUEST); + } + let response = client + .delete(&url) + .bearer_auth("secret") + .json(&json!({"id":id})) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let mark: Value = response.json().await.unwrap(); + assert_eq!(mark["reclaim_after"], "2200"); + assert_eq!( + client + .get(&url) + .bearer_auth("secret") + .send() + .await + .unwrap() + .status(), + StatusCode::NOT_FOUND + ); + let list: Value = client + .get(url.rsplit_once('/').unwrap().0) + .bearer_auth("secret") + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(list["entries"], json!([])); + let retry: Value = client + .delete(&url) + .bearer_auth("secret") + .json(&json!({"id":id})) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(retry, mark); + let status: Value = client + .get(format!("{url}/deletion/{id}")) + .bearer_auth("secret") + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(status["deletion"], mark); + drop(client); + service.stop().await; +} + +#[tokio::test] +async fn real_background_worker_obeys_one_second_grace() { + let store = Arc::new(TestStore::default()); + let authority = Arc::new(DatasetAuthority::new(store).with_reclaim_grace_seconds(1)); + let name = DatasetIdentity::new(NamespacePath::root(), "short-grace").unwrap(); + let id = authority.create_dataset(name.clone()).await.unwrap(); + let cancel = ReadCancellation::new(); + let mark = authority.delete_dataset(&name, id, &cancel).await.unwrap(); + assert!(!mark.complete); + let worker = crowdb_access_server::dataset::start_dataset_cleanup_for_tests(authority.clone(), None); + tokio::time::timeout(std::time::Duration::from_secs(3), async { + loop { + if authority + .dataset_deletion(&name, id) + .await + .unwrap() + .unwrap() + .complete + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + }) + .await + .unwrap(); + worker.abort(); + let _ = worker.await; + authority.create_dataset(name).await.unwrap(); +} diff --git a/app/crowdb-access-server/tests/dataset_identity_contract_test.rs b/app/crowdb-access-server/tests/dataset_identity_contract_test.rs new file mode 100644 index 000000000..1fd7666e2 --- /dev/null +++ b/app/crowdb-access-server/tests/dataset_identity_contract_test.rs @@ -0,0 +1,91 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. +#[path = "common/dataset.rs"] +mod dataset; +use crowdb_access_dataset::{ + DatasetDeletionSweep, DatasetIdentity, NamespacePath, OperationId, ReadCancellation, +}; +use dataset::{TestDatasetHttp, TestStore}; +use reqwest::Client; +use serde_json::json; +use std::sync::{atomic::Ordering, Arc}; + +#[tokio::test] +async fn dataset_identity_header_fences_stale_editors_and_validates_before_reservation() { + let test = TestDatasetHttp::start_payload(Arc::new(TestStore::default())).await; + let client = Client::new(); + let name = DatasetIdentity::new(NamespacePath::root(), "reused").unwrap(); + let old = test.authority.create_dataset(name.clone()).await.unwrap(); + let url = format!( + "{}/api/dataset/v2/namespaces/dataset-ns/datasets/reused", + test.origin + ); + let head = format!("{url}/head"); + let response = client + .get(&head) + .bearer_auth("secret") + .header("x-crowdb-dataset-id", old.to_string()) + .send() + .await + .unwrap(); + assert_eq!(response.status(), 200, "{}", response.text().await.unwrap()); + for value in ["0".repeat(32), "A".repeat(32), "bad".into()] { + let response = client + .get(&head) + .bearer_auth("secret") + .header("x-crowdb-dataset-id", value) + .send() + .await + .unwrap(); + assert_eq!(response.status(), 400, "{}", response.text().await.unwrap()); + } + let response = client + .get(&head) + .bearer_auth("secret") + .header("x-crowdb-dataset-id", old.to_string()) + .header("x-crowdb-dataset-id", old.to_string()) + .send() + .await + .unwrap(); + assert_eq!(response.status(), 400, "{}", response.text().await.unwrap()); + test.authority + .delete_dataset(&name, old, &ReadCancellation::new()) + .await + .unwrap(); + test.clock.store(2200, Ordering::Release); + test.authority + .reclaim_datasets_page(&mut DatasetDeletionSweep::default(), &ReadCancellation::new()) + .await + .unwrap(); + let current = test.authority.create_dataset(name.clone()).await.unwrap(); + let response = client + .post(format!("{url}/uploads")) + .bearer_auth("secret") + .header("x-crowdb-dataset-id", old.to_string()) + .json( + &json!({"id":OperationId::random().to_string(),"length":"4","part_size":4,"md5":"1".repeat(32)}), + ) + .send() + .await + .unwrap(); + assert_eq!(response.status(), 409); + assert_eq!( + response.json::().await.unwrap()["code"], + "dataset_replaced" + ); + for id in [None, Some(current.to_string())] { + let mut request = client.get(&head).bearer_auth("secret"); + if let Some(id) = id { + request = request.header("x-crowdb-dataset-id", id); + } + let response = request.send().await.unwrap(); + assert_eq!(response.status(), 200, "{}", response.text().await.unwrap()); + } + // Rejected stale uploads must not make the replacement Dataset nonempty. + test.authority + .delete_dataset(&name, current, &ReadCancellation::new()) + .await + .unwrap(); + drop(client); + test.stop().await; +} diff --git a/app/crowdb-access-server/tests/dataset_publication_cost_test.rs b/app/crowdb-access-server/tests/dataset_publication_cost_test.rs index 85617fc09..5cabdf541 100644 --- a/app/crowdb-access-server/tests/dataset_publication_cost_test.rs +++ b/app/crowdb-access-server/tests/dataset_publication_cost_test.rs @@ -67,7 +67,8 @@ async fn inline_publication_and_identical_replay_have_bounded_metadata_cost() { let after = counts(&store); assert_eq!( (after.0 - before.0, after.1 - before.1), - (12, 11 + 3 * u64::from(samples)) + // Dataset deletion admission adds two reads and one root CAS. + (14, 12 + 3 * u64::from(samples)) ); println!( "cold publication: {} reads, {} mutations", diff --git a/app/crowdb-access-server/tests/dataset_split_preview_test.rs b/app/crowdb-access-server/tests/dataset_split_preview_test.rs new file mode 100644 index 000000000..844a22724 --- /dev/null +++ b/app/crowdb-access-server/tests/dataset_split_preview_test.rs @@ -0,0 +1,108 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. +#[path = "common/dataset_v2.rs"] +#[allow(dead_code)] +mod common; +#[path = "common/dataset.rs"] +mod dataset; +use base64::{engine::general_purpose::STANDARD, Engine}; +use common::response; +use dataset::{TestDatasetHttp, TestStore}; +use reqwest::Client; +use serde_json::{json, Value}; +use std::{ + collections::BTreeSet, + sync::{atomic::Ordering, Arc}, +}; + +#[tokio::test] +async fn split_preview_scans_empty_windows_binds_rules_and_never_mutates() { + let store = Arc::new(TestStore::default()); + let test = TestDatasetHttp::start(store.clone()).await; + let client = Client::new(); + let (root, snapshot) = common::fixture(&client, &test.origin).await; + let url = format!("{root}/snapshots/{snapshot}/split-preview"); + let input = json!({"name":"late", "origin":{"kind":"evaluated"}, "definition":{"kind":"selection", "predicate":{"kind":"prefix", "value":STANDARD.encode([40])}}}); + let writes = store.mutations.load(Ordering::Relaxed); + let first = response(client.post(&url).json(&input), 200).await; + assert_eq!(first["entries"], json!([])); + assert!(first["after"].is_string()); + let mut changed = input.clone(); + changed["definition"]["predicate"]["value"] = json!(STANDARD.encode([41])); + response( + client + .post(&url) + .query(&[("after", first["after"].as_str().unwrap())]) + .json(&changed), + 400, + ) + .await; + let second = response( + client + .post(&url) + .query(&[("after", first["after"].as_str().unwrap())]) + .json(&input), + 200, + ) + .await; + assert_eq!(second["entries"], json!([])); + let third = response( + client + .post(&url) + .query(&[("after", second["after"].as_str().unwrap())]) + .json(&input), + 200, + ) + .await; + assert_eq!(third["entries"], json!([STANDARD.encode([40, 0, 255])])); + assert!(third["after"].is_null()); + response(client.post(&url).query(&[("limit", "101")]).json(&input), 400).await; + let mut invalid = common::definitions()[0].clone(); + invalid["definition"]["index"] = json!(99); + response(client.post(&url).json(&invalid), 400).await; + assert_eq!(store.mutations.load(Ordering::Relaxed), writes); + drop(client); + test.stop().await; +} + +#[tokio::test] +async fn preview_and_committed_weighted_membership_match() { + let store = Arc::new(TestStore::default()); + let test = TestDatasetHttp::start(store.clone()).await; + let client = Client::new(); + let (root, snapshot) = common::fixture(&client, &test.origin).await; + let base = format!("{root}/snapshots/{snapshot}"); + let writes = store.mutations.load(Ordering::Relaxed); + let split = common::definitions()[0].clone(); + let mut preview = BTreeSet::new(); + let mut after = Value::Null; + loop { + let mut request = client.post(format!("{base}/split-preview")).json(&split); + if let Some(cursor) = after.as_str() { + request = request.query(&[("after", cursor)]); + } + let page = response(request, 200).await; + for id in page["entries"].as_array().unwrap() { + preview.insert(id.as_str().unwrap().to_owned()); + } + after = page["after"].clone(); + if after.is_null() { + break; + } + } + assert_eq!(store.mutations.load(Ordering::Relaxed), writes); + let revision = "12121212121212121212121212121212"; + common::prepare(&client, &root, &snapshot, revision, false).await; + let url = format!("{base}/split-revisions/{revision}/splits/train%2Fa%20b/members"); + let members = response(client.get(&url).query(&[("limit", "100")]), 200).await; + let committed: BTreeSet<_> = members["entries"] + .as_array() + .unwrap() + .iter() + .map(|id| id.as_str().unwrap().to_owned()) + .collect(); + assert_eq!(preview, committed); + assert!(!preview.is_empty()); + drop(client); + test.stop().await; +} diff --git a/app/crowdb-access-server/tests/dataset_v2_contract_test.rs b/app/crowdb-access-server/tests/dataset_v2_contract_test.rs index f6fed5b2b..13d29b413 100644 --- a/app/crowdb-access-server/tests/dataset_v2_contract_test.rs +++ b/app/crowdb-access-server/tests/dataset_v2_contract_test.rs @@ -38,7 +38,10 @@ async fn capabilities_use_existing_auth_and_match_configured_operations() { "bounded-inspection", "inspection-preparation", "dataset-management", + "dataset-delete", + "dataset-identity-fence", "split-revisions", + "split-preview", "split-revision-delete", "split-preparation", "split-head-update", diff --git a/app/crowdb-access-server/tests/iceberg_maintenance_http_test.rs b/app/crowdb-access-server/tests/iceberg_maintenance_http_test.rs new file mode 100644 index 000000000..9280002d0 --- /dev/null +++ b/app/crowdb-access-server/tests/iceberg_maintenance_http_test.rs @@ -0,0 +1,178 @@ +#[path = "common/iceberg_file_blocks.rs"] +#[allow(dead_code)] +mod blocks; +#[path = "common/iceberg_store.rs"] +mod common; +#[path = "common/iceberg_table_http.rs"] +#[allow(dead_code)] +mod fixture; + +use fixture::TestTableHttp; +use reqwest::Method; +use serde_json::{json, Value}; + +const TABLE: &str = "/v1/namespaces/analytics/tables/events"; + +async fn value(response: reqwest::Response, status: u16) -> Value { + let actual = response.status(); + let text = response.text().await.unwrap(); + assert_eq!(actual.as_u16(), status, "{text}"); + serde_json::from_str(&text).unwrap() +} + +async fn setup() -> (TestTableHttp, Value) { + let fixture = TestTableHttp::writable().await; + let created = value( + fixture + .post( + "/v1/namespaces/analytics/tables", + "w", + None, + &json!({"name":"events","schema":{"type":"struct","schema-id":0,"fields":[]}}), + ) + .await, + 200, + ) + .await; + let now = u64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(), + ) + .unwrap(); + let snapshot = |id, sequence| { + json!({"action":"add-snapshot","snapshot":{ + "snapshot-id":id,"sequence-number":sequence,"timestamp-ms":now,"schema-id":0, + "summary":{"operation":"append"},"manifest-list":format!("{}/metadata/{id}.avro", created["metadata"]["location"].as_str().unwrap())}}) + }; + let current = value( + fixture + .post( + TABLE, + "w", + None, + &json!({"requirements":[],"updates":[ + snapshot(10,1), snapshot(20,2), + {"action":"set-snapshot-ref","ref-name":"main","type":"branch","snapshot-id":20}]}), + ) + .await, + 200, + ) + .await; + (fixture, current) +} + +fn expire(base: &Value, id: i64) -> Value { + json!({"x-crowdb-base-metadata-location":base["metadata-location"],"requirements":[ + {"type":"assert-table-uuid","uuid":base["metadata"]["table-uuid"]}], + "updates":[{"action":"remove-snapshots","snapshot-ids":[id]}]}) +} + +#[tokio::test] +async fn guarded_expiry_protects_current_and_new_tags_then_allows_unreferenced_history() { + let (fixture, before) = setup().await; + let config = value(fixture.request(Method::GET, "/v1/config", "w", None).await, 200).await; + assert_eq!(config["overrides"]["crowdb.guarded-maintenance"], "true"); + let error = value(fixture.post(TABLE, "w", None, &expire(&before, 20)).await, 409).await; + assert!(error["error"]["message"].as_str().unwrap().contains("referenced")); + let tag = json!({"requirements":[],"updates":[{"action":"set-snapshot-ref","ref-name":"keep","type":"tag","snapshot-id":10}]}); + let tagged = value(fixture.post(TABLE, "w", None, &tag).await, 200).await; + value(fixture.post(TABLE, "w", None, &expire(&before, 10)).await, 409).await; + value(fixture.post(TABLE, "w", None, &expire(&tagged, 10)).await, 409).await; + assert_eq!( + value(fixture.request(Method::GET, TABLE, "r", None).await, 200).await, + tagged + ); + let cleared = value( + fixture + .post( + TABLE, + "w", + None, + &json!({"requirements":[],"updates":[{"action":"remove-snapshot-ref","ref-name":"keep"}]}), + ) + .await, + 200, + ) + .await; + let expired = value(fixture.post(TABLE, "w", None, &expire(&cleared, 10)).await, 200).await; + assert_eq!(expired["metadata"]["snapshots"].as_array().unwrap().len(), 1); + assert_eq!(expired["metadata"]["current-snapshot-id"], 20); + fixture.finish().await; +} + +#[tokio::test] +async fn guarded_commit_recovers_lost_reply_but_keeps_later_replay_uncertain() { + let (fixture, before) = setup().await; + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(); + let identity = format!("{:08x}-{:04x}-7000-8000-000000000001", now >> 16, now & 0xffff); + let body = expire(&before, 10); + fixture + .store + .lose_reply_kind + .store(5, std::sync::atomic::Ordering::SeqCst); + value(fixture.post(TABLE, "w", Some(&identity), &body).await, 503).await; + let expired = value(fixture.post(TABLE, "w", Some(&identity), &body).await, 200).await; + assert_eq!(expired["metadata"]["snapshots"].as_array().unwrap().len(), 1); + let later = value( + fixture + .post( + TABLE, + "w", + None, + &json!({"requirements":[],"updates":[{"action":"set-properties","updates":{"later":"yes"}}]}), + ) + .await, + 200, + ) + .await; + value(fixture.post(TABLE, "w", Some(&identity), &body).await, 503).await; + assert_eq!( + value(fixture.request(Method::GET, TABLE, "r", None).await, 200).await, + later + ); + fixture.finish().await; +} + +#[tokio::test] +async fn guarded_retry_can_publish_prepared_metadata_and_a_late_ref_wins_the_head_cas() { + for concurrent_ref in [false, true] { + let (fixture, before) = setup().await; + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(); + let identity = format!("{:08x}-{:04x}-7000-8000-000000000002", now >> 16, now & 0xffff); + let body = expire(&before, 10); + fixture + .store + .pause_head_cas + .store(true, std::sync::atomic::Ordering::SeqCst); + let (first, other) = tokio::join!(fixture.post(TABLE, "w", Some(&identity), &body), async { + fixture.store.head_cas_entered.notified().await; + let response = if concurrent_ref { + fixture.post(TABLE, "w", None, &json!({"requirements":[],"updates":[ + {"action":"set-snapshot-ref","ref-name":"new-branch","type":"branch","snapshot-id":10}]})).await + } else { + fixture.post(TABLE, "w", Some(&identity), &body).await + }; + fixture.store.head_cas_release.notify_one(); + response + }); + value(other, 200).await; + value(first, if concurrent_ref { 409 } else { 200 }).await; + let current = value(fixture.request(Method::GET, TABLE, "r", None).await, 200).await; + assert_eq!( + current["metadata"]["snapshots"].as_array().unwrap().len(), + if concurrent_ref { 2 } else { 1 } + ); + if concurrent_ref { + assert_eq!(current["metadata"]["refs"]["new-branch"]["snapshot-id"], 10); + } + fixture.finish().await; + } +} diff --git a/app/crowdb-chunk-kv-server/src/main.rs b/app/crowdb-chunk-kv-server/src/main.rs index 7a2eeb751..bd8edb6d2 100644 --- a/app/crowdb-chunk-kv-server/src/main.rs +++ b/app/crowdb-chunk-kv-server/src/main.rs @@ -569,6 +569,7 @@ async fn main() -> std::process::ExitCode { info!("chunk KV service admission drained"); }) .await; + info!("chunk KV HTTP shutdown complete"); if let Err(error) = &serving { error!(%error, "HTTP management server failed"); } @@ -576,18 +577,26 @@ async fn main() -> std::process::ExitCode { materialization_task.abort(); transition_task.abort(); refresh_task.abort(); - let _ = tokio::join!( + // These workers only observe the service and have already been fenced by + // begin_drain. Drop their handles after cancellation: a worker may be in + // a synchronous storage observation, and waiting for that observation here + // would hold the management restart request open until the supervisor kills + // the process. The service drain below is the durable shutdown barrier. + drop(( heartbeat_task, materialization_task, transition_task, - refresh_task - ); + refresh_task, + )); + info!("chunk KV background workers stopped"); if let Err(error) = service.drain_writers().await { warn!(%error, "WAL writer drain incomplete; restart must retain lease exclusion"); } + info!("chunk KV WAL writer drain complete"); if let Err(error) = process_lease.withdraw_observation().await { warn!(%error, "failed to unregister chunk KV instance"); } + info!("chunk KV observation withdrawal complete"); if let Some(runner) = &mut metrics_runner { runner.stop().await; } diff --git a/app/crowdb-chunkdb/src/lifecycle/handler/seal.rs b/app/crowdb-chunkdb/src/lifecycle/handler/seal.rs index dcd237f40..c2e798843 100644 --- a/app/crowdb-chunkdb/src/lifecycle/handler/seal.rs +++ b/app/crowdb-chunkdb/src/lifecycle/handler/seal.rs @@ -9,6 +9,7 @@ use super::{ StripReservationState, }; use crowdb_protocol::chunkdb::rpc::{ChunkStrip, ChunkType, StripReservationGroup}; +use std::collections::HashSet; use tracing::info; impl LifecycleHandler { @@ -67,6 +68,15 @@ impl LifecycleHandler { return Err(LifecycleError::StateConflict); } let current_state = ChunkState::from_proto(chunk.state); + if current_state == ChunkState::Sealed && chunk.acknowledged_cursor == seal_bytes { + // Metadata publication can outlive a failed block cleanup or reply. + // Resume the exact seal without changing its durable byte boundary. + self.finish_seal_cleanup(&mut chunk).await?; + if let Some(ref mut g) = guard { + g.refresh(chunk.clone()); + } + return Ok(chunk); + } current_state.check_can_seal()?; validate_seal_bounds(&chunk, seal_bytes, seal_length)?; @@ -103,23 +113,41 @@ impl LifecycleHandler { if let Some(ref mut g) = guard { g.refresh(chunk.clone()); } - if let Some(operation_id) = cleanup_operation { + self.finish_seal_cleanup(&mut chunk).await?; + if let Some(ref mut g) = guard { + g.refresh(chunk.clone()); + } + info!(chunk_id = ?chunk_id, seal_length, "chunk sealed"); + Ok(chunk) + } + + async fn finish_seal_cleanup(&self, chunk: &mut Chunk) -> Result<(), LifecycleError> { + let now_ms = unix_time_ms(); + let current_segments: HashSet<_> = chunk.strips.iter().flat_map(extract_segments).collect(); + let mut pending = Vec::new(); + for intent in &chunk.cleanup_intents { + if intent.not_before_ms > now_ms { + pending.push(intent.clone()); + continue; + } + let retired = intent + .retired_segments + .iter() + .filter(|segment| !current_segments.contains(segment)) + .copied() + .collect(); self.allocator .pool() - .free_blocks(unused_segments) + .free_blocks(retired) .await .map_err(LifecycleError::Cleanup)?; - chunk - .cleanup_intents - .retain(|intent| intent.operation_id != Some(operation_id)); + } + if pending.len() != chunk.cleanup_intents.len() { + chunk.cleanup_intents = pending; chunk.modify_ts = chunk.modify_ts.saturating_add(1); - self.store.put_chunk(&chunk).await?; - if let Some(ref mut g) = guard { - g.refresh(chunk.clone()); - } + self.store.put_chunk(chunk).await?; } - info!(chunk_id = ?chunk_id, seal_length, "chunk sealed"); - Ok(chunk) + Ok(()) } async fn cancel_seal_reservations(&self, chunk_id: &ChunkId) -> Result { diff --git a/app/crowdb-chunkdb/tests/stream_seal_test.rs b/app/crowdb-chunkdb/tests/stream_seal_test.rs index e3b4fdec1..54867c819 100644 --- a/app/crowdb-chunkdb/tests/stream_seal_test.rs +++ b/app/crowdb-chunkdb/tests/stream_seal_test.rs @@ -4,9 +4,10 @@ mod common; use common::cluster::{seed_hardware, ChunkdbHarness, DiskdbServer, KvCluster}; -use crowdb_chunkdb::lifecycle::LifecycleError; +use crowdb_chunkdb::lifecycle::{LifecycleError, LifecycleHandler}; use crowdb_protocol::chunk_stream::StreamName; use crowdb_protocol::chunkdb::rpc::{ChunkState, ChunkType, StripType}; +use std::sync::Arc; #[tokio::test] async fn stream_and_wal_seal_cannot_shorten_a_persisted_byte_cursor() { @@ -124,3 +125,64 @@ async fn exact_empty_seal_records_zero_bytes_and_zero_physical_capacity() { assert_eq!(sealed.capacity, 0); assert!(sealed.strips.is_empty()); } + +#[tokio::test] +async fn exact_seal_resumes_cleanup_after_block_owner_discovery_failure() { + let cluster = KvCluster::start_pinned().await; + seed_hardware(&cluster.make_hardware_client()).await; + let _diskdb = DiskdbServer::start(&cluster).await; + let harness = ChunkdbHarness::start(&cluster).await; + let chunk = harness + .handler + .allocate_chunk_owned( + None, + 4, + 2, + StripType::Mirror, + 0, + 0, + 2, + ChunkType::Wal, + 9, + 60_000, + StreamName { high: 89, low: 97 }.chunk_owner_key(), + ) + .await + .unwrap(); + let id = chunk.id.unwrap(); + harness.pool.update_disk_id_lookup(&[]); + let error = harness.handler.seal_chunk_bytes(&id, 17).await.unwrap_err(); + assert!(matches!(error, LifecycleError::Cleanup(_)), "{error}"); + let published = harness.handler.query_chunk(&id).await.unwrap(); + assert_eq!(published.state, ChunkState::Sealed as i32); + assert_eq!(published.acknowledged_cursor, 17); + assert_eq!(published.cleanup_intents.len(), 1); + assert!(!published.cleanup_intents[0].retired_segments.is_empty()); + + // A changed byte boundary and a stale observed snapshot must not be accepted. + for cursor in [16, 18] { + assert!(harness.handler.seal_chunk_bytes(&id, cursor).await.is_err()); + assert_eq!(harness.handler.query_chunk(&id).await.unwrap(), published); + } + assert!(matches!( + harness.handler.seal_observed_chunk(&chunk, 17).await, + Err(LifecycleError::StateConflict) + )); + + harness + .pool + .update_disk_id_lookup(&harness.topology.snapshot().disk_groups()); + let sealed = harness.handler.seal_chunk_bytes(&id, 17).await.unwrap(); + assert_eq!(sealed.acknowledged_cursor, 17); + assert_eq!(sealed.strips, published.strips); + assert_eq!(sealed.sealed_ts_ms, published.sealed_ts_ms); + assert!(sealed.cleanup_intents.is_empty()); + assert!(sealed.modify_ts > published.modify_ts); + assert_eq!(harness.handler.seal_chunk_bytes(&id, 17).await.unwrap(), sealed); + let cold = LifecycleHandler::new( + Arc::clone(&harness.store), + Arc::clone(&harness.allocator), + harness.topology.clone(), + ); + assert_eq!(cold.seal_chunk_bytes(&id, 17).await.unwrap(), sealed); +} diff --git a/app/crowdb-diskdb/Cargo.toml b/app/crowdb-diskdb/Cargo.toml index 55a33a79b..3f0451ab8 100644 --- a/app/crowdb-diskdb/Cargo.toml +++ b/app/crowdb-diskdb/Cargo.toml @@ -163,3 +163,7 @@ path = "tests/retained_recovery_test.rs" [[test]] name = "online_activation_test" path = "tests/online_activation_test.rs" + +[[test]] +name = "compaction_schedule_test" +path = "tests/compaction_schedule_test.rs" diff --git a/app/crowdb-diskdb/src/recovery/compaction.rs b/app/crowdb-diskdb/src/recovery/compaction.rs index 728a4f266..a7285d074 100644 --- a/app/crowdb-diskdb/src/recovery/compaction.rs +++ b/app/crowdb-diskdb/src/recovery/compaction.rs @@ -4,9 +4,9 @@ //! Snapshot compaction engine (R73 strategy 3) + preparatory thread. //! //! `CompactionEngine` runs as a background task, periodically -//! compacting zones whose `uncompacted_free_record_count` exceeds the -//! threshold. `PreparatoryThread` pre-compacts the next batch of zones -//! in the rotation order so rotation is instant. Both call the shared +//! compacting non-active zones when their free-record backlog reaches the +//! threshold or the full sweep cadence expires. `PreparatoryThread` pre-compacts +//! the next batch of zones in the rotation order so rotation is instant. Both call the shared //! `compact_zone` function. //! //! Crash safety: the `ZoneValue` snapshot + free-record deletes are one @@ -29,6 +29,10 @@ use crate::model::records::ZoneRecords; use crate::model::zone::DdbZone; use crate::recovery::ZoneLoadError; +mod schedule; +#[cfg(feature = "test-util")] +pub use schedule::CompactionSchedule as CompactionScheduleForTests; + /// RAII guard that clears the `compacting` flag on drop. Ensures the /// flag is cleared even on panic or early return (HIGH-5). struct CompactingGuard<'a>(&'a AtomicBool); @@ -209,10 +213,11 @@ pub fn matching_free_records_for_tests( pub struct CompactionEngine { kv: Arc, config: CompactionConfig, + started: std::time::Instant, + schedule: schedule::CompactionSchedule, /// Optional shared config handle for live-apply of the timer - /// cadence. When set, `trigger()` returns `TimerFn` reading - /// `persistence.compaction_cadence_secs` from this handle each - /// tick; when `None`, falls back to the fixed snapshot. + /// cadence. Polling checks the threshold at most every five seconds; + /// the independent sweep deadline comes from the current persistence config. config_handle: Option>>, } @@ -223,6 +228,8 @@ impl CompactionEngine { Self { kv, config, + started: std::time::Instant::now(), + schedule: schedule::CompactionSchedule::default(), config_handle: None, } } @@ -312,7 +319,13 @@ impl CompactionEngine { impl BackgroundTask for CompactionEngine { fn run_cycle<'a>(&'a self, ctx: &'a BgCtx) -> CycleFut<'a> { Box::pin(async move { - let threshold = ctx.config.load().persistence.snapshot_compaction_threshold; + let config = ctx.config.load(); + let threshold = self.schedule.threshold( + self.started.elapsed().as_secs(), + u64::from(config.persistence.compaction_cadence_secs), + config.persistence.snapshot_compaction_threshold, + ); + drop(config); self.compaction_cycle(&ctx.container, threshold, &ctx.metrics) .await; Ok(()) @@ -325,11 +338,15 @@ impl BackgroundTask for CompactionEngine { let handle = Arc::clone(handle); Trigger::TimerFn(Box::new(move || { std::time::Duration::from_secs(u64::from( - handle.load().persistence.compaction_cadence_secs, + handle.load().persistence.compaction_cadence_secs.min(5), )) })) } - None => Trigger::Timer(self.config.compaction_cadence), + None => Trigger::Timer( + self.config + .compaction_cadence + .min(std::time::Duration::from_secs(5)), + ), } } diff --git a/app/crowdb-diskdb/src/recovery/compaction/schedule.rs b/app/crowdb-diskdb/src/recovery/compaction/schedule.rs new file mode 100644 index 000000000..632c14c18 --- /dev/null +++ b/app/crowdb-diskdb/src/recovery/compaction/schedule.rs @@ -0,0 +1,22 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::atomic::{AtomicU64, Ordering}; + +/// Threshold polls must not postpone the periodic sweep of small backlogs. +#[derive(Default)] +pub struct CompactionSchedule { + last_sweep_seconds: AtomicU64, +} + +impl CompactionSchedule { + pub fn threshold(&self, elapsed_seconds: u64, cadence_seconds: u64, threshold: u32) -> u32 { + let last = self.last_sweep_seconds.load(Ordering::Acquire); + if elapsed_seconds.saturating_sub(last) >= cadence_seconds { + self.last_sweep_seconds.store(elapsed_seconds, Ordering::Release); + 1 + } else { + threshold.max(1) + } + } +} diff --git a/app/crowdb-diskdb/tests/compaction_schedule_test.rs b/app/crowdb-diskdb/tests/compaction_schedule_test.rs new file mode 100644 index 000000000..cd4716c72 --- /dev/null +++ b/app/crowdb-diskdb/tests/compaction_schedule_test.rs @@ -0,0 +1,27 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_diskdb::recovery::compaction::CompactionScheduleForTests; + +#[test] +fn frequent_threshold_polls_do_not_starve_small_zone_backlogs() { + let schedule = CompactionScheduleForTests::default(); + for elapsed in (5..300).step_by(5) { + assert_eq!(schedule.threshold(elapsed, 300, 4096), 4096); + } + // A 256-block zone cannot accumulate 4096 distinct frees. The deadline + // must still admit its smallest nonempty backlog to the compaction pass. + assert_eq!(schedule.threshold(300, 300, 4096), 1); + assert_eq!(schedule.threshold(305, 300, 4096), 4096); + assert_eq!(schedule.threshold(600, 300, 4096), 1); +} + +#[test] +fn live_cadence_changes_and_delayed_wakeups_keep_periodic_cleanup() { + let schedule = CompactionScheduleForTests::default(); + assert_eq!(schedule.threshold(20, 300, 64), 64); + assert_eq!(schedule.threshold(30, 30, 64), 1); + assert_eq!(schedule.threshold(1000, 30, 64), 1); + assert_eq!(schedule.threshold(1005, 60, 128), 128); + assert_eq!(schedule.threshold(1060, 60, 128), 1); +} diff --git a/app/crowdb-diskdb/tests/config_reload_test.rs b/app/crowdb-diskdb/tests/config_reload_test.rs index 04103c4da..95d3abc3a 100644 --- a/app/crowdb-diskdb/tests/config_reload_test.rs +++ b/app/crowdb-diskdb/tests/config_reload_test.rs @@ -100,12 +100,12 @@ fn compaction_engine_trigger_uses_timer_fn_when_config_handle_set() { let engine = CompactionEngine::new(Arc::clone(&ddb_kv), cfg).with_config_handle(Arc::clone(&handle)); match engine.trigger() { Trigger::TimerFn(f) => { - assert_eq!(f(), Duration::from_secs(30)); + assert_eq!(f(), Duration::from_secs(5)); // Simulate a config reload. let mut new_config = DdbConfig::default(); - new_config.persistence.compaction_cadence_secs = 15; + new_config.persistence.compaction_cadence_secs = 2; handle.store(Arc::new(new_config)); - assert_eq!(f(), Duration::from_secs(15)); + assert_eq!(f(), Duration::from_secs(2)); } _other => panic!("expected TimerFn"), } @@ -123,7 +123,7 @@ fn compaction_engine_trigger_uses_fixed_timer_without_config_handle() { }; let engine = CompactionEngine::new(Arc::clone(&ddb_kv), cfg); match engine.trigger() { - Trigger::Timer(d) => assert_eq!(d, Duration::from_secs(120)), + Trigger::Timer(d) => assert_eq!(d, Duration::from_secs(5)), _other => panic!("expected Timer"), } } diff --git a/app/crowdb-web/src/access.rs b/app/crowdb-web/src/access.rs index ce5704db9..d1f0bda16 100644 --- a/app/crowdb-web/src/access.rs +++ b/app/crowdb-web/src/access.rs @@ -20,6 +20,7 @@ use crate::state::AppState; mod credentials; pub(crate) mod dataset; +mod iceberg_files; mod inspection; mod signing; @@ -179,9 +180,16 @@ pub(crate) async fn configure( fn request_headers(source: &HeaderMap) -> HeaderMap { let mut headers = HeaderMap::new(); - for name in ["content-type", "range", "if-match", "if-none-match"] { - if let Some(value) = source.get(name) { - headers.insert(name, value.clone()); + for name in [ + "content-type", + "range", + "if-match", + "if-none-match", + "idempotency-key", + "x-crowdb-dataset-id", + ] { + for value in source.get_all(name) { + headers.append(name, value.clone()); } } headers @@ -193,6 +201,9 @@ pub(crate) async fn proxy( request: Request, ) -> Result { let protocol = ¶meters[0]; + if protocol == "iceberg-files" { + return iceberg_files::proxy(state, request).await; + } if protocol == "dataset" { return dataset::proxy(state, request).await; } diff --git a/app/crowdb-web/src/access/dataset.rs b/app/crowdb-web/src/access/dataset.rs index 4be78cfb5..17ed478f1 100644 --- a/app/crowdb-web/src/access/dataset.rs +++ b/app/crowdb-web/src/access/dataset.rs @@ -14,7 +14,7 @@ const PREFIX: &str = "/api/access/dataset"; const API: &str = "/api/dataset/v2"; const METADATA_LIMIT: usize = 1024 * 1024; -/// The exact matrix POST is an observation, not a state change. +/// Exact matrix and Split preview POSTs observe immutable Snapshot content. pub(crate) fn observes(method: &Method, path: &str) -> bool { if matches!(*method, Method::GET | Method::HEAD | Method::OPTIONS) { return true; @@ -25,7 +25,7 @@ pub(crate) fn observes(method: &Method, path: &str) -> bool { let parts: Vec<_> = path.split('/').collect(); *method == Method::POST && matches!(parts.as_slice(), - ["", "namespaces", "dataset-ns", "datasets", dataset, "snapshots", snapshot, "matrix"] + ["", "namespaces", "dataset-ns", "datasets", dataset, "snapshots", snapshot, "matrix" | "split-preview"] if !dataset.is_empty() && snapshot.len() == 32 && snapshot.bytes().all(|b| b.is_ascii_hexdigit())) } diff --git a/app/crowdb-web/src/access/iceberg_files.rs b/app/crowdb-web/src/access/iceberg_files.rs new file mode 100644 index 000000000..b36ee64f6 --- /dev/null +++ b/app/crowdb-web/src/access/iceberg_files.rs @@ -0,0 +1,234 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. +//! Table-scoped `FileIO` through the configured Catalog origin and fresh writer grants. +use super::{credentials, signing, ApiError, BODY_LIMIT}; +use crate::{ + error::{err_400, err_502}, + state::AppState, +}; +use axum::{body::Body, extract::Request, response::Response}; +use serde_json::Value; + +fn header<'a>(request: &'a Request, name: &str) -> Result<&'a str, ApiError> { + if request.headers().get_all(name).iter().count() != 1 { + return Err(err_400("Expected one file scope header")); + } + request + .headers() + .get(name) + .and_then(|value| value.to_str().ok()) + .filter(|value| value.len() <= 8192) + .ok_or_else(|| err_400("Invalid file scope header")) +} + +fn credential_url(origin: &str, endpoint: &str) -> Result { + let (path, _) = endpoint + .split_once('?') + .ok_or_else(|| err_400("Missing table identity"))?; + let url = reqwest::Url::parse(&format!("{origin}{endpoint}")) + .map_err(|_| err_400("Invalid credential endpoint"))?; + let parts: Vec<_> = path.split('/').collect(); + if parts.len() != 7 + || parts[..3] != ["", "v1", "namespaces"] + || parts[4] != "tables" + || parts[6] != "credentials" + || [parts[3], parts[5]].iter().any(|value| { + value.is_empty() || *value == "." || *value == ".." || value.contains('#') || value.contains('\\') + }) + { + return Err(err_400("Invalid table credential scope")); + } + let mut seen = std::collections::BTreeSet::new(); + for (name, value) in url.query_pairs() { + if !seen.insert(name.to_string()) { + return Err(err_400("Duplicate credential selector")); + } + let valid = match name.as_ref() { + "table-id" | "storage-namespace-id" => { + value.len() == 32 + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + && value.bytes().any(|byte| byte != b'0') + } + "format-version" => matches!(value.as_ref(), "1" | "2" | "3"), + _ => false, + }; + if !valid { + return Err(err_400("Invalid credential selector")); + } + } + if !seen.contains("table-id") { + return Err(err_400("Missing immutable table identity")); + } + if url.origin().ascii_serialization() != origin || url.path() != path || url.fragment().is_some() { + return Err(err_400("Invalid credential endpoint")); + } + Ok(url) +} + +async fn grant( + client: &reqwest::Client, + state: &AppState, + target: &str, + endpoint: &str, +) -> Result { + let token = credentials::writer(state).map_err(err_502)?; + let mut response = client + .get(credential_url(target, endpoint)?) + .bearer_auth(token) + .send() + .await + .map_err(|_| err_502("Could not refresh table credentials"))?; + let status = response.status(); + let mut bytes = Vec::new(); + while let Some(chunk) = response + .chunk() + .await + .map_err(|_| err_502("Could not read table credentials"))? + { + if bytes.len() + chunk.len() > 65536 { + return Err(err_502("Table credentials exceed 64 KiB")); + } + bytes.extend_from_slice(&chunk); + } + if !status.is_success() { + return Err(( + status, + axum::Json(crate::error::ErrorBody { + error: format!( + "Table credential refresh failed: {}", + String::from_utf8_lossy(&bytes) + ), + }), + )); + } + serde_json::from_slice(&bytes).map_err(|_| err_502("Invalid table credentials")) +} + +fn file_query(request: &Request) -> Result { + if request.headers().get_all("x-crowdb-file-query").iter().count() > 1 { + return Err(err_400("Duplicate file query header")); + } + let query = request + .headers() + .get("x-crowdb-file-query") + .map(|value| value.to_str()) + .transpose() + .map_err(|_| err_400("Invalid file query"))? + .unwrap_or(""); + if query.len() > 2048 { + return Err(err_400("File query exceeds its limit")); + } + let mut names = std::collections::BTreeSet::new(); + let file_query = reqwest::Url::parse(&format!("http://file.invalid/?{query}")) + .map_err(|_| err_400("Invalid file query"))?; + for (name, _) in file_query.query_pairs() { + if !matches!( + name.as_ref(), + "crowdb-native-upload" + | "uploads" + | "uploadId" + | "partNumber" + | "part-number-marker" + | "max-parts" + ) || !names.insert(name.to_string()) + { + return Err(err_400("Unsupported file query")); + } + } + Ok(query.to_owned()) +} + +fn file_location(location: &str) -> Result { + let file = reqwest::Url::parse(location).map_err(|_| err_400("Invalid file location"))?; + if file.scheme() != "s3" + || file.host_str().is_none() + || !file.username().is_empty() + || file.password().is_some() + || file.query().is_some() + || file.fragment().is_some() + || file.as_str() != location + { + return Err(err_400("Expected a canonical S3 file location")); + } + Ok(file) +} + +fn file_credentials(grant: &Value, location: &str) -> Result { + let grants = grant["storage-credentials"] + .as_array() + .filter(|values| values.len() == 1) + .ok_or_else(|| err_502("Invalid table credential grant"))?; + let prefix = grants[0]["prefix"] + .as_str() + .ok_or_else(|| err_502("Missing table credential prefix"))?; + if !location + .strip_prefix(&format!("{}/", prefix.trim_end_matches('/'))) + .is_some_and(|tail| !tail.is_empty()) + { + return Err(err_400("File is outside the authorized table")); + } + let config = &grants[0]["config"]; + let value = |name: &str| { + config[name] + .as_str() + .map(str::to_owned) + .ok_or_else(|| err_502("Incomplete file credentials")) + }; + Ok(credentials::S3Credentials { + access: value("s3.access-key-id")?, + secret: value("s3.secret-access-key")?, + session: Some(value("s3.session-token")?), + region: "us-east-1".into(), + }) +} + +pub(super) async fn proxy(state: AppState, request: Request) -> Result { + if !matches!( + request.method().as_str(), + "GET" | "HEAD" | "PUT" | "POST" | "DELETE" + ) { + return Err(err_400("Unsupported file method")); + } + let target = super::origin(&state, "iceberg")?.ok_or_else(|| err_502("Catalog endpoint unavailable"))?; + let endpoint = header(&request, "x-crowdb-file-credentials")?; + let location = header(&request, "x-crowdb-file-location")?; + let query = file_query(&request)?; + let file = file_location(location)?; + let client = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .timeout(std::time::Duration::from_secs(300)) + .build() + .map_err(|reason| err_502(reason.to_string()))?; + let grant = grant(&client, &state, &target, endpoint).await?; + let credentials = file_credentials(&grant, location)?; + let mut url = reqwest::Url::parse(&format!("{target}/{}{}", file.host_str().unwrap(), file.path())) + .map_err(|_| err_400("Invalid native file URL"))?; + if !query.is_empty() { + url.set_query(Some(&query)); + } + let method = request.method().clone(); + let mut headers = super::request_headers(request.headers()); + let body = axum::body::to_bytes(request.into_body(), BODY_LIMIT) + .await + .map_err(|_| err_400("File part exceeds 16 MiB; use multipart upload"))?; + signing::sign(&credentials, &method, &url, &body, &mut headers).map_err(err_502)?; + let upstream = client + .request(method, url) + .headers(headers) + .body(body) + .send() + .await + .map_err(|_| err_502("File transfer reply unavailable; retain the saved input before retrying"))?; + let mut response = Response::builder().status(upstream.status()); + for name in ["content-type", "content-length", "etag", "content-range"] { + if let Some(value) = upstream.headers().get(name) { + response = response.header(name, value); + } + } + response + .body(Body::from_stream(upstream.bytes_stream())) + .map_err(|reason| err_502(reason.to_string())) +} diff --git a/app/crowdb-web/src/diskdb/proxy.rs b/app/crowdb-web/src/diskdb/proxy.rs index f0ca07056..63097731b 100644 --- a/app/crowdb-web/src/diskdb/proxy.rs +++ b/app/crowdb-web/src/diskdb/proxy.rs @@ -24,7 +24,7 @@ use crowdb_protocol::diskdb::rpc::{ }; use crate::error::{err_400, err_404, err_502, ErrorBody}; -use crate::mgmt::{build_hardware_client, rpc_endpoint_for_node}; +use crate::mgmt::build_hardware_client; use crate::state::AppState; // ── lazy DiskdbClient init ─────────────────────────────────────── @@ -33,35 +33,9 @@ use crate::state::AppState; /// `build_hardware_client`). Returns `None` when no group-0 endpoint /// is known. pub(crate) async fn build_diskdb_client(state: &AppState) -> Option { - let snap = state.monitor_cache.snapshot().await; - if snap.is_empty() && !state.managed_mode { - return None; - } - // Use the shared kv_client so topology discovery seeds are - // consistent with the rest of the web server. Creating a standalone - // client with empty seeds here caused "no seeds configured" errors - // when refresh_endpoints triggered a topology refresh. - let kv = state.kv_client().await; - // Prefer the live group-0 leader endpoint from the monitor cache - // (the actual crowdb-rpc listener for store 0 on the leader node). - // Fall back to per-node rpc_endpoint_for_node for any running node. - if let Some(ep) = state.monitor_cache.group0_leader_endpoint().await { - let bare = crate::mgmt::strip_scheme(crate::mgmt::remap_zero_host(&ep)); - kv.seed_leader(0, 0, bare); - } else { - for node_id in snap.keys() { - // Skip stopped servers — no runtime pid means the process is - // not running. Contacting it only wastes time on - // connection-refused. - if state.runtime_pid(*node_id).is_none() { - continue; - } - if let Some(ep) = rpc_endpoint_for_node(state, *node_id, 0).await { - kv.seed_leader(0, 0, ep); - break; - } - } - } + // Resolve the same authority as capacity inventory. Admitted services may + // be remote and absent from the legacy process monitor cache. + build_hardware_client(state).await?; // Use the shared ServiceDiscoveryClient's ServiceRegistryClient so // the discovery cache + connection pool are shared with the rest // of the web server (R128). diff --git a/app/crowdb-web/src/lib.rs b/app/crowdb-web/src/lib.rs index 8cdd235ef..47522a48b 100644 --- a/app/crowdb-web/src/lib.rs +++ b/app/crowdb-web/src/lib.rs @@ -33,6 +33,8 @@ mod standalone; pub mod state; pub use node::observation::{start_node_observer, NodeObserver}; +#[cfg(feature = "test-util")] +pub use services::observation::remote_access_health_for_tests; pub use state::AppState; #[cfg(feature = "test-util")] diff --git a/app/crowdb-web/src/node/auto_join.rs b/app/crowdb-web/src/node/auto_join.rs index 9e7bfd3b9..22ad23963 100644 --- a/app/crowdb-web/src/node/auto_join.rs +++ b/app/crowdb-web/src/node/auto_join.rs @@ -6,7 +6,7 @@ mod formation; mod runner; -use std::fs; +use std::{collections::BTreeMap, fs}; use axum::{extract::State, http::StatusCode, Json}; use serde::{Deserialize, Serialize}; @@ -28,6 +28,8 @@ struct Policy { target_cluster_id: Option, #[serde(default)] local_draft_selected: bool, + #[serde(default)] + candidates: BTreeMap, } impl Default for Policy { @@ -37,6 +39,7 @@ impl Default for Policy { enabled: true, target_cluster_id: None, local_draft_selected: false, + candidates: BTreeMap::new(), } } } @@ -45,16 +48,10 @@ fn read(state: &AppState) -> Result { let path = state.runtime_root.join("auto-join.json"); let metadata = match fs::symlink_metadata(&path) { Ok(metadata) => metadata, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - return Ok(Policy { - enabled: deployment(state)?.mode - == crowdb_protocol::mgmt::node::deployment::DeploymentMode::Local, - ..Policy::default() - }) - } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Policy::default()), Err(error) => return Err(err_502(error.to_string())), }; - if !metadata.is_file() || metadata.len() > 4096 { + if !metadata.is_file() || metadata.len() > 65536 { return Err(err_409("Invalid Auto join policy")); } let policy: Policy = serde_json::from_slice(&fs::read(path).map_err(|error| err_502(error.to_string()))?) @@ -105,7 +102,7 @@ pub(crate) async fn status(State(state): State) -> Result, Err(error) => return Err(err_502(error.to_string())), }; Ok(Json( - json!({"enabled": policy.enabled, "target_cluster_id": policy.target_cluster_id, + json!({"enabled": policy.enabled, "candidates": policy.candidates, "target_cluster_id": policy.target_cluster_id, "mode": "standard", "deployment_mode": deployment.mode, "cluster": deployment.cluster, "required_voters": deployment.initial_voters, "observation": observation}), )) } @@ -115,6 +112,7 @@ pub(crate) async fn status(State(state): State) -> Result, pub(crate) struct Update { enabled: bool, target: Option, + discovery_id: Option, } #[derive(Deserialize)] @@ -132,7 +130,7 @@ pub(crate) async fn update( return Err(err_400("Workspace is readonly")); } let mut resources = vec!["node/auto-join".into()]; - if body.target.is_some() { + if body.target.is_some() || body.discovery_id.is_some() { resources.extend(["cluster/init".into(), "node/admission".into()]); } let _operation = crate::services::Operation::claim(&state, resources)?; @@ -162,7 +160,38 @@ pub(crate) async fn update( policy.target_cluster_id = None; policy.local_draft_selected = true; } - policy.enabled = body.enabled; + if let Some(id) = body.discovery_id { + if body.target.is_some() || uuid::Uuid::parse_str(&id).is_err() { + return Err(err_400( + "Candidate Auto join requires a discovery UUID and no target override", + )); + } + let snapshot = super::candidates(State(state.clone())) + .await + .map_err(|_| err_502("Discovery unavailable"))? + .0; + let candidate = snapshot + .candidates + .iter() + .find(|node| node.advertisement.discovery_id == id) + .ok_or_else(|| err_409("Candidate is no longer discovered"))?; + if candidate.state != crowdb_protocol::mgmt::node::CandidateState::Unbound { + return Err(err_409("Auto join can only be changed for an unbound candidate")); + } + if body.enabled + && candidate.advertisement.formation.as_ref() != Some(&deployment(&state)?.formation()) + { + return Err(err_409( + "Candidate formation differs from this cluster; review admission manually", + )); + } + if policy.candidates.len() >= 256 && !policy.candidates.contains_key(&id) { + return Err(err_409("Candidate Auto join policy is full")); + } + policy.candidates.insert(id, body.enabled); + } else { + policy.enabled = body.enabled; + } super::save(&state.runtime_root.join("auto-join.json"), &policy)?; status(State(state.clone())).await } diff --git a/app/crowdb-web/src/node/auto_join/runner.rs b/app/crowdb-web/src/node/auto_join/runner.rs index f83841a06..ab4ae78e5 100644 --- a/app/crowdb-web/src/node/auto_join/runner.rs +++ b/app/crowdb-web/src/node/auto_join/runner.rs @@ -57,7 +57,7 @@ async fn step( node::save(&state.runtime_root.join("auto-join.json"), &policy)?; } } - if !policy.enabled { + if !policy.enabled && !policy.candidates.values().any(|enabled| *enabled) { return Ok(json!({"phase": "disabled"})); } let authority = node::status(State(state.clone())).await.0; @@ -93,9 +93,7 @@ async fn admit_candidates( attempts: &mut BTreeMap, bound: bool, ) -> Result { - if bound { - return finish(state, true, Vec::new(), Vec::new()).await; - } + let policy = super::read(state)?; let deployment = super::deployment(state)?; let snapshot = node::candidates(State(state.clone())) .await @@ -108,9 +106,37 @@ async fn admit_candidates( ) })? .0; - let selected = super::formation::candidates(&snapshot, &deployment)?; - if let Some(waiting) = super::formation::waiting_for_peers(state, &snapshot, &selected, &deployment)? { - return Ok(waiting); + let selected: Vec<_> = if bound { + snapshot + .candidates + .iter() + .filter(|candidate| { + candidate.state == crowdb_protocol::mgmt::node::CandidateState::Unbound + && policy + .candidates + .get(&candidate.advertisement.discovery_id) + .copied() + .unwrap_or(policy.enabled) + && candidate.advertisement.formation.as_ref() == Some(&deployment.formation()) + }) + .collect() + } else { + super::formation::candidates(&snapshot, &deployment)? + .into_iter() + .filter(|candidate| { + policy + .candidates + .get(&candidate.advertisement.discovery_id) + .copied() + .unwrap_or(policy.enabled) + }) + .collect() + }; + if !bound { + if let Some(waiting) = super::formation::waiting_for_peers(state, &snapshot, &selected, &deployment)? + { + return Ok(waiting); + } } super::formation::prepare_local_rack(state, &deployment).await?; let config = state @@ -181,11 +207,29 @@ async fn finish( json!({"phase": "waiting", "members": eligible, "required_voters": required, "needs_access": needs_access}), ); } - if !super::read(state)?.enabled { - return Ok(json!({"phase": "disabled"})); + let _operation = crate::services::Operation::claim( + state, + vec![ + "cluster/init".into(), + "node/admission".into(), + "node/auto-join".into(), + ], + )?; + let policy = super::read(state)?; + let records = node::records(state)?; + if eligible.iter().any(|id| { + !records.iter().any(|record| { + record.node_id == *id + && !record.cancelled + && policy + .candidates + .get(&record.discovery_id) + .copied() + .unwrap_or(policy.enabled) + }) + }) { + return Ok(json!({"phase": "waiting", "reason": "Candidate Auto join selection changed"})); } - let _operation = - crate::services::Operation::claim(state, vec!["cluster/init".into(), "node/admission".into()])?; let progress = node::initialization::trigger(state, &eligible, Some(false)).await?; Ok(json!({"phase": "initializing", "members": eligible, "operation": progress})) } @@ -222,7 +266,8 @@ async fn admit_candidate( setup: None, expected_formation: Some(deployment.formation()), }; - if !super::read(state)?.enabled { + let policy = super::read(state)?; + if !policy.candidates.get(id).copied().unwrap_or(policy.enabled) { return Ok(Some(json!({"phase": "disabled"}))); } match node::admit(State(state.clone()), Json(body)).await { diff --git a/app/crowdb-web/src/node/storage/files.rs b/app/crowdb-web/src/node/storage/files.rs index 9b6cc9f5d..7e1ca10ca 100644 --- a/app/crowdb-web/src/node/storage/files.rs +++ b/app/crowdb-web/src/node/storage/files.rs @@ -11,7 +11,7 @@ use axum::{ use crowdb_console_shared::deployment::admission::remote_control; use crowdb_protocol::mgmt::node::{ storage::{ - FileDiskAdmission, FilePreparation, FilePreparationPage, PrepareDiskFile, + managed_file_zone_bytes, FileDiskAdmission, FilePreparation, FilePreparationPage, PrepareDiskFile, DEFAULT_FILE_CAPACITY_BYTES, DEFAULT_FILE_RESERVE_BYTES, }, NodeControl, @@ -26,7 +26,6 @@ use crate::{ use super::{remote_node, Failure}; const UNIT_SIZE: u32 = 1024 * 1024; -const ZONE_SIZE: u64 = 32 * 1024 * 1024 * 1024; #[derive(Deserialize)] #[serde(deny_unknown_fields)] @@ -64,15 +63,8 @@ pub(crate) async fn create( ) -> Result<(StatusCode, Json), Failure> { let node = remote_node(&state, node_id)?; canonical_operation(&body.operation_id)?; - if body.capacity_bytes == 0 - || body.capacity_bytes > i64::MAX as u64 - || body.capacity_bytes % (64 * u64::from(UNIT_SIZE)) != 0 - || (body.capacity_bytes > ZONE_SIZE && body.capacity_bytes % ZONE_SIZE != 0) - { - return Err(err_400( - "File capacity must be a positive multiple of 64 MiB within signed 64-bit bounds; above 32 GiB it must contain whole 32 GiB zones", - )); - } + let zone_size_bytes = managed_file_zone_bytes(body.capacity_bytes) + .ok_or_else(|| err_400("File capacity must be a positive multiple of 64 MiB within signed 64-bit bounds; above 32 GiB it must contain whole 32 GiB zones"))?; let binding = crate::node::cluster_binding(&state)? .ok_or_else(|| err_409("File disk creation is waiting for Group 0"))?; let request = PrepareDiskFile { @@ -86,7 +78,7 @@ pub(crate) async fn create( rack_id: node.rack_id, disk_group_id: body.disk_group_id, unit_size_bytes: UNIT_SIZE, - zone_size_bytes: ZONE_SIZE.min(body.capacity_bytes), + zone_size_bytes, media_type: None, }), }; diff --git a/app/crowdb-web/src/services/observation.rs b/app/crowdb-web/src/services/observation.rs index 6cbb729c6..aa0f2fea2 100644 --- a/app/crowdb-web/src/services/observation.rs +++ b/app/crowdb-web/src/services/observation.rs @@ -1,6 +1,10 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +mod access; +#[cfg(feature = "test-util")] +pub use access::remote_access_health_for_tests; + use crate::state::AppState; use axum::{extract::State, Json}; use crowdb_console_shared::{cluster::NodeHealth, config::ServiceType}; @@ -72,6 +76,7 @@ pub async fn http_list_servers(State(state): State) -> Json { @@ -96,7 +101,7 @@ pub async fn http_list_servers(State(state): State) -> Json, snap: &std::collections::BTreeMap, + intents: &[crowdb_protocol::mgmt::node::NodeServiceIntent], ) -> NodeHealth { if s.service_type == ServiceType::AccessServer { - if let Some(url) = cfg - .local_launches - .get(&s.id) - .and_then(crowdb_console_shared::config::LocalLaunchSpec::access_health_url) - { - let healthy = reqwest::Client::new() - .get(url) - .timeout(std::time::Duration::from_secs(1)) - .send() - .await - .is_ok_and(|response| response.status().is_success()); - if healthy { - NodeHealth::Up - } else { - NodeHealth::Down - } - } else { - NodeHealth::Unknown - } + access::observe(cfg, s, state.node_monitor_url.is_some(), intents).await } else if s.service_type == ServiceType::PaxosKv && pid.is_some() && s.node_id.is_some_and(|id| { diff --git a/app/crowdb-web/src/services/observation/access.rs b/app/crowdb-web/src/services/observation/access.rs new file mode 100644 index 000000000..48ef887bb --- /dev/null +++ b/app/crowdb-web/src/services/observation/access.rs @@ -0,0 +1,88 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Access uses its dedicated readiness listener, including remote deployments. + +use crowdb_console_shared::{cluster::NodeHealth, config::ServerEntry, ConsoleConfig}; +use crowdb_protocol::mgmt::node::{NodeServiceAction, NodeServiceIntent}; + +pub(super) async fn observe( + config: &ConsoleConfig, + service: &ServerEntry, + remote: bool, + intents: &[NodeServiceIntent], +) -> NodeHealth { + let url = if remote { + let Some(intent) = intents.iter().find(|intent| { + intent.kind == "access" + && intent.service_id == service.id + && Some(intent.node_id) == service.node_id + }) else { + return NodeHealth::Unknown; + }; + if matches!(intent.action, NodeServiceAction::Stop | NodeServiceAction::Delete) { + return NodeHealth::Down; + } + config + .node(intent.node_id) + .and_then(|node| remote_url(&intent.configuration, &node.host)) + } else { + config + .local_launches + .get(&service.id) + .and_then(crowdb_console_shared::config::LocalLaunchSpec::access_health_url) + }; + probe(url.as_deref()).await +} + +fn remote_url(configuration: &str, host: &str) -> Option { + let config: toml::Value = toml::from_str(configuration).ok()?; + let mut address: std::net::SocketAddr = config.get("health")?.get("listen")?.as_str()?.parse().ok()?; + let owner: std::net::IpAddr = host.parse().ok()?; + if address.port() == 0 || (!address.ip().is_unspecified() && address.ip() != owner) { + return None; + } + for protocol in ["s3", "iceberg", "dataset"] { + if config + .get(protocol) + .and_then(|value| value.get("listen")) + .and_then(toml::Value::as_str) + .and_then(|value| value.parse::().ok()) + .is_some_and(|listener| listener.port() == address.port()) + { + return None; + } + } + address.set_ip(owner); + Some(format!("http://{address}/_crowdb/health/ready")) +} + +async fn probe(url: Option<&str>) -> NodeHealth { + let Some(url) = url else { + return NodeHealth::Unknown; + }; + let Ok(client) = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .timeout(std::time::Duration::from_secs(1)) + .build() + else { + return NodeHealth::Unknown; + }; + if client + .get(url) + .send() + .await + .is_ok_and(|response| response.status().is_success()) + { + NodeHealth::Up + } else { + NodeHealth::Down + } +} + +/// Exercise remote URL validation and the same readiness probe as the server inventory. +#[cfg(feature = "test-util")] +pub async fn remote_access_health_for_tests(configuration: &str, host: &str) -> NodeHealth { + probe(remote_url(configuration, host).as_deref()).await +} diff --git a/app/crowdb-web/tests/access_routes_test.rs b/app/crowdb-web/tests/access_routes_test.rs index d3f2e19f1..9810236ba 100644 --- a/app/crowdb-web/tests/access_routes_test.rs +++ b/app/crowdb-web/tests/access_routes_test.rs @@ -174,6 +174,64 @@ fn configured(origin: &str) -> axum::Router { ) } +#[tokio::test] +async fn iceberg_proxy_preserves_client_retry_identity_and_duplicate_header_validation() { + const KEY: &str = "019a2f6c-0000-7000-8000-000000000001"; + let upstream = axum::Router::new().route( + "/*path", + any(|request: Request| async move { + let keys: Vec<_> = request + .headers() + .get_all("idempotency-key") + .iter() + .map(|value| value.to_str().unwrap().to_owned()) + .collect(); + if keys.len() != 1 { + return (StatusCode::BAD_REQUEST, "duplicate retry identity".to_owned()); + } + (StatusCode::OK, keys[0].clone()) + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!("http://{}", listener.local_addr().unwrap()); + let task = tokio::spawn(async move { + axum::serve(listener, upstream).await.unwrap(); + }); + let app = configured(&origin); + for duplicate in [false, true] { + let mut builder = Request::builder() + .method("POST") + .uri("/api/access/iceberg/v1/namespaces/example/tables/events") + .header("idempotency-key", KEY); + if duplicate { + builder = builder.header("idempotency-key", KEY); + } + let response = app + .clone() + .oneshot(builder.body(Body::from("{}")).unwrap()) + .await + .unwrap(); + assert_eq!( + response.status(), + if duplicate { + StatusCode::BAD_REQUEST + } else { + StatusCode::OK + } + ); + let body = axum::body::to_bytes(response.into_body(), 100).await.unwrap(); + assert_eq!( + body, + if duplicate { + "duplicate retry identity" + } else { + KEY + } + ); + } + task.abort(); +} + #[tokio::test] async fn remote_deployment_resolves_persisted_origins_without_local_launches() { let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("remote-access-origin"); diff --git a/app/crowdb-web/tests/auto_formation_test.rs b/app/crowdb-web/tests/auto_formation_test.rs index ecc7d568b..37a9e5ce3 100644 --- a/app/crowdb-web/tests/auto_formation_test.rs +++ b/app/crowdb-web/tests/auto_formation_test.rs @@ -76,11 +76,18 @@ async fn known_bound_scope_and_duplicate_identity_require_explicit_attention() { } #[tokio::test] -async fn production_does_not_enable_local_convenience_admission() { - let fixture = TestFormationFixture::start(policy(3, DeploymentMode::Production), vec![], 1).await; - let report = fixture.observation().await; - assert_eq!(report["phase"], "disabled"); - fixture.assert_no_admission(); +async fn production_requires_complete_formation_and_an_explicit_rack() { + for (peer_count, phase) in [(0, "waiting"), (2, "needs-rack")] { + let policy = policy(3, DeploymentMode::Production); + let peers = vec![(policy.formation(), CandidateState::Unbound); peer_count]; + let fixture = + TestFormationFixture::with_password(policy, peers, 1, Some((b"local-test-password", 0o600))) + .await; + let report = fixture.observation().await; + assert_eq!(report["phase"], phase, "{report}"); + assert!(fixture.state.config.read().unwrap().racks.is_empty()); + fixture.assert_no_admission(); + } } #[tokio::test] diff --git a/app/crowdb-web/tests/auto_join_test.rs b/app/crowdb-web/tests/auto_join_test.rs index 128da7d21..6597a4d40 100644 --- a/app/crowdb-web/tests/auto_join_test.rs +++ b/app/crowdb-web/tests/auto_join_test.rs @@ -331,3 +331,84 @@ async fn completed_cleanup_retires_console_credentials_before_another_cluster_is "observing an acknowledged cleanup must preserve the new cluster's credentials" ); } + +#[tokio::test] +async fn candidate_choices_are_independent_durable_and_do_not_enable_other_nodes() { + use axum::{routing::get, Json}; + use crowdb_protocol::mgmt::node::{ + CandidateNode, CandidateSnapshot, CandidateState, NodeAdvertisement, NodeHandshake, NodeHardware, + NODE_PROTOCOL_VERSION, + }; + let workspace = TestWorkspace::new(); + let ids = [ + "00000000-0000-4000-8000-000000000001", + "00000000-0000-4000-8000-000000000002", + ]; + let candidates: Vec<_> = ids + .iter() + .map(|id| CandidateNode { + advertisement: NodeAdvertisement { + discovery_id: (*id).into(), + protocol_version: NODE_PROTOCOL_VERSION, + monitor_endpoints: vec![], + cluster_id: None, + formation: Some(workspace.formation()), + }, + state: CandidateState::Unbound, + }) + .collect(); + let local = NodeHandshake { + advertisement: candidates[0].advertisement.clone(), + physical_host_id: "host".into(), + rack_hint: None, + hardware: NodeHardware::default(), + }; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let monitor = Router::new() + .route("/node", get(move || async move { Json(local) })) + .route( + "/candidates", + get(move || async move { + Json(CandidateSnapshot { + nodes: candidates, + discovery_diagnostics: vec![], + }) + }), + ); + let server = tokio::spawn(async move { + axum::serve(listener, monitor).await.unwrap(); + }); + let state = || workspace.state().with_node_monitor(&endpoint).unwrap(); + assert_eq!( + request(router(state()), Some(json!({"enabled":false}))).await.0, + StatusCode::OK + ); + let (status, body) = request( + router(state()), + Some(json!({"enabled":true,"discovery_id":ids[0]})), + ) + .await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["enabled"], false); + assert_eq!(body["candidates"][ids[0]], true); + assert!(body["candidates"][ids[1]].is_null()); + let (status, _) = request( + router(state()), + Some(json!({"enabled":false,"discovery_id":ids[1]})), + ) + .await; + assert_eq!(status, StatusCode::OK); + let (_, restarted) = request(router(state()), None).await; + assert_eq!(restarted["candidates"][ids[0]], true); + assert_eq!(restarted["candidates"][ids[1]], false); + let (status, _) = request( + router(state()), + Some(json!({"enabled":true,"discovery_id":"00000000-0000-4000-8000-000000000003"})), + ) + .await; + assert_eq!(status, StatusCode::CONFLICT); + assert!(!workspace.0.join("console/prepared-bootstrap.json").exists()); + server.abort(); + let _ = server.await; +} diff --git a/app/crowdb-web/tests/dataset_proxy_test.rs b/app/crowdb-web/tests/dataset_proxy_test.rs index c159a0318..a13fb4c8c 100644 --- a/app/crowdb-web/tests/dataset_proxy_test.rs +++ b/app/crowdb-web/tests/dataset_proxy_test.rs @@ -100,6 +100,7 @@ async fn readonly_allows_exact_matrix_but_rejects_all_mutations() { let snapshot = "11111111111111111111111111111111"; for (method, suffix, status) in [ ("POST", format!("/snapshots/{snapshot}/matrix"), 200), + ("POST", format!("/snapshots/{snapshot}/split-preview"), 200), ("GET", format!("/snapshots/{snapshot}/contains"), 200), ("POST", "/compositions".into(), 403), ("PUT", format!("/snapshots/{snapshot}"), 403), @@ -112,6 +113,7 @@ async fn readonly_allows_exact_matrix_but_rejects_all_mutations() { ("PUT", String::new(), 403), ("DELETE", "/uploads/123".into(), 403), ("POST", format!("/snapshots/{snapshot}/matrix/extra"), 403), + ("POST", format!("/snapshots/{snapshot}/split-preview/extra"), 403), ] { let response = app .clone() diff --git a/app/crowdb-web/tests/diskdb_binding_test.rs b/app/crowdb-web/tests/diskdb_binding_test.rs new file mode 100644 index 000000000..c9e179224 --- /dev/null +++ b/app/crowdb-web/tests/diskdb_binding_test.rs @@ -0,0 +1,45 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/cluster_target.rs"] +mod cluster_target; +#[path = "common/memory_kv.rs"] +mod memory_kv; + +use axum::{body::Body, http::Request}; +use cluster_target::TestClusterTarget; +use tower::ServiceExt; + +#[tokio::test] +async fn disk_drilldown_uses_persisted_binding_without_local_process_observation() { + let fixture = TestClusterTarget::start().await; + assert_eq!(fixture.prepared.identity, fixture.binding.bootstrap); + assert!(fixture.root.path().join("node-binding.json").exists()); + assert!(fixture + .kv + .get( + 0, + 0, + b"/deployment/cluster", + crowdb_kv_client::ReadMode::Linearizable, + None + ) + .await + .is_ok()); + assert!(fixture.state.monitor_cache.snapshot().await.is_empty()); + let response = crowdb_web::router(fixture.state.clone()) + .oneshot( + Request::builder() + .uri("/api/diskdb/usage?dg=1") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + // This fixture has no DiskDB owner. Authority resolution still succeeds; + // the request must reach DiskDB routing rather than claim no cluster exists. + let body = axum::body::to_bytes(response.into_body(), 8192).await.unwrap(); + let text = String::from_utf8(body.to_vec()).unwrap(); + assert!(!text.contains("cluster not initialized"), "{text}"); + assert!(fixture.state.diskdb_client.read().await.is_some()); +} diff --git a/app/crowdb-web/tests/iceberg_file_proxy_test.rs b/app/crowdb-web/tests/iceberg_file_proxy_test.rs new file mode 100644 index 000000000..6fe217ca8 --- /dev/null +++ b/app/crowdb-web/tests/iceberg_file_proxy_test.rs @@ -0,0 +1,59 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. +use axum::response::IntoResponse; +use axum::{ + body::Body, + http::{Request, StatusCode}, + routing::any, +}; +use crowdb_console_shared::{config::ServerEntry, ConsoleConfig}; +use crowdb_web::{router, AppState}; +use tower::ServiceExt; +const PREFIX: &str = "s3://iceberg/11111111111111111111111111111111/22222222222222222222222222222222/33333333333333333333333333333333/"; +const CREDENTIALS: &str = "/v1/namespaces/ns/tables/events/credentials?table-id=33333333333333333333333333333333&storage-namespace-id=22222222222222222222222222222222&format-version=2"; + +#[tokio::test] +async fn files_use_configured_origin_fresh_writer_grants_and_authorized_table_prefix() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("iceberg-file-proxy"); + crowdb_monitor::ServerCredentials::load_or_create(root.path()).unwrap(); + let upstream = axum::Router::new().route("/*path", any(|request: Request| async move { + if request.uri().path().ends_with("/credentials") { + assert!(request.headers()["authorization"].to_str().unwrap().starts_with("Bearer ")); + assert!(request.uri().query().unwrap().contains("table-id=33333333333333333333333333333333")); + return (StatusCode::OK, axum::Json(serde_json::json!({"storage-credentials":[{"prefix":PREFIX,"config":{ + "s3.access-key-id":"temporary-key","s3.secret-access-key":"temporary-secret","s3.session-token":"temporary-session" + }}]}))).into_response(); + } + assert!(request.uri().path().ends_with("/data/new.parquet")); + assert_eq!(request.headers()["x-amz-security-token"], "temporary-session"); + assert!(request.headers()["authorization"].to_str().unwrap().contains("Credential=temporary-key/")); + assert!(!request.headers().contains_key("x-crowdb-file-credentials")); + assert_eq!(axum::body::to_bytes(request.into_body(), 100).await.unwrap(), "PAR1"); + (StatusCode::OK, [("etag", "file-etag")], "uploaded").into_response() + })); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!("http://{}", listener.local_addr().unwrap()); + let task = tokio::spawn(async move { + axum::serve(listener, upstream).await.unwrap(); + }); + let mut config = ConsoleConfig::default(); + config + .servers + .push(ServerEntry::new("console-access-iceberg", origin)); + let app = router(AppState::with_runtime_root(config, root.path().to_owned())); + for (credential, location, expected) in [ + (CREDENTIALS.to_owned(), format!("{PREFIX}data/new.parquet"), StatusCode::OK), + (CREDENTIALS.to_owned(), format!("{}data/new.parquet", PREFIX.replace("33333333", "44444444")), StatusCode::BAD_REQUEST), + ("http://untrusted.invalid/v1/namespaces/ns/tables/events/credentials?table-id=33333333333333333333333333333333".into(), format!("{PREFIX}data/new.parquet"), StatusCode::BAD_REQUEST), + (format!("{CREDENTIALS}&cleanup=true"), format!("{PREFIX}data/new.parquet"), StatusCode::BAD_REQUEST), + (CREDENTIALS.to_owned(), "s3://iceberg/../outside".into(), StatusCode::BAD_REQUEST), + ] { + let response = app.clone().oneshot(Request::builder().method("PUT").uri("/api/access/iceberg-files") + .header("x-crowdb-file-credentials", credential).header("x-crowdb-file-location", location) + .header("authorization", "browser-secret").body(Body::from("PAR1")).unwrap()).await.unwrap(); + let status = response.status(); + let bytes = axum::body::to_bytes(response.into_body(), 65536).await.unwrap(); + assert_eq!(status, expected, "{}", String::from_utf8_lossy(&bytes)); + } + task.abort(); +} diff --git a/app/crowdb-web/tests/service_health_test.rs b/app/crowdb-web/tests/service_health_test.rs index 3d339f619..5f13141e6 100644 --- a/app/crowdb-web/tests/service_health_test.rs +++ b/app/crowdb-web/tests/service_health_test.rs @@ -153,3 +153,39 @@ async fn node_mode_never_interprets_a_cached_pid_in_the_web_host_namespace() { Some(std::process::id()) ); } + +#[tokio::test] +async fn remote_access_uses_committed_health_listener_without_local_launch_metadata() { + use crowdb_console_shared::cluster::NodeHealth; + use crowdb_web::remote_access_health_for_tests; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let app = axum::Router::new().route("/_crowdb/health/ready", axum::routing::get(|| async { "ready" })); + let task = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + let config = format!("[health]\nlisten = '{address}'\n[iceberg]\nlisten = '127.0.0.1:1'\n"); + assert_eq!( + remote_access_health_for_tests(&config, "127.0.0.1").await, + NodeHealth::Up + ); + assert_eq!( + remote_access_health_for_tests(&config, "192.0.2.1").await, + NodeHealth::Unknown + ); + let shared = format!("[health]\nlisten = '{address}'\n[s3]\nlisten = '{address}'\n"); + assert_eq!( + remote_access_health_for_tests(&shared, "127.0.0.1").await, + NodeHealth::Unknown + ); + assert_eq!( + remote_access_health_for_tests("[iceberg]\nlisten = '127.0.0.1:1'", "127.0.0.1").await, + NodeHealth::Unknown + ); + task.abort(); + let _ = task.await; + assert_eq!( + remote_access_health_for_tests(&config, "127.0.0.1").await, + NodeHealth::Down + ); +} diff --git a/app/crowdb-web/ui/package-lock.json b/app/crowdb-web/ui/package-lock.json index 084e319c7..23f93d2d2 100644 --- a/app/crowdb-web/ui/package-lock.json +++ b/app/crowdb-web/ui/package-lock.json @@ -9,6 +9,7 @@ "version": "0.3.0", "dependencies": { "clsx": "^2.1.1", + "hyparquet": "1.31.5", "lucide-react": "^0.456.0", "react": "^18.3.1", "react-dom": "^18.3.1", @@ -4277,6 +4278,12 @@ "node": ">=16.17.0" } }, + "node_modules/hyparquet": { + "version": "1.31.5", + "resolved": "https://registry.npmjs.org/hyparquet/-/hyparquet-1.31.5.tgz", + "integrity": "sha512-KPVfaxW8xOiO2/4fjeiBfFNEXhGVw9tfcb9ha2iNZdJ3J/yBjEwIeAnTyyuhllFFTb9YO9wg0DfRAOnm6KCvBg==", + "license": "MIT" + }, "node_modules/iconv-lite": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", diff --git a/app/crowdb-web/ui/package.json b/app/crowdb-web/ui/package.json index 9648a2f33..6b0a3a3eb 100644 --- a/app/crowdb-web/ui/package.json +++ b/app/crowdb-web/ui/package.json @@ -16,6 +16,7 @@ }, "dependencies": { "clsx": "^2.1.1", + "hyparquet": "1.31.5", "lucide-react": "^0.456.0", "react": "^18.3.1", "react-dom": "^18.3.1", diff --git a/app/crowdb-web/ui/src/App.test.tsx b/app/crowdb-web/ui/src/App.test.tsx index df1f6955b..17811833a 100644 --- a/app/crowdb-web/ui/src/App.test.tsx +++ b/app/crowdb-web/ui/src/App.test.tsx @@ -5,6 +5,11 @@ import { afterEach, expect, it, vi } from 'vitest'; import App from './App'; import { Domain } from './types'; import * as api from './api'; +const capacityRequests = vi.hoisted(() => vi.fn()); +const diskdbInstances = vi.hoisted(() => []); +vi.mock('./shell/NodeCandidatesContext', () => ({ CandidateProvider: ({ children }: { children: import('react').ReactNode }) => children, + useNodeCandidates: () => ({ snapshot: undefined, candidates: [] }) })); +vi.mock('./shell/CandidateNodes', () => ({ CandidateNodes: () => null })); vi.mock('./api', async importOriginal => ({ ...await importOriginal(), @@ -19,10 +24,10 @@ vi.mock('./data/useLogicalTree', () => ({ useLogicalTree: () => ({ stores: [{ store_id: 0, name: 'System', groups: [], nodes: [] }], groups: [], loading: false, error: null, refresh: vi.fn(), }) })); -vi.mock('./data/useCapacityTree', () => ({ useCapacityTree: () => ({ - instances: [], usage: null, hardwareCapacity: null, scanStatus: null, loading: false, +vi.mock('./data/useCapacityTree', () => ({ useCapacityTree: (options: unknown) => { capacityRequests(options); return ({ + instances: diskdbInstances, usage: null, hardwareCapacity: null, scanStatus: null, loading: false, error: null, refresh: vi.fn(), -}) })); +}); } })); vi.mock('./services/useNodeServicePlans', () => ({ useNodeServicePlans: () => ({ plans: {}, stop: vi.fn() }) })); vi.mock('./views/ClusterView', () => ({ ClusterView: () => null })); vi.mock('./views/KvView', () => ({ KvView: () => null })); @@ -32,7 +37,20 @@ vi.mock('./chunk-kv/ChunkKvView', () => ({ ChunkKvView: () => null })); vi.mock('./views/IcebergView', () => ({ IcebergView: () => null })); vi.mock('./views/S3View', () => ({ S3View: () => null })); -afterEach(() => { vi.restoreAllMocks(); vi.unstubAllGlobals(); vi.resetAllMocks(); }); +afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); vi.unstubAllGlobals(); vi.resetAllMocks(); }); + +it('updates service health without waiting for a capacity observation or browser reload', async () => { + vi.useFakeTimers(); + vi.mocked(api.nodeDeploymentStatus).mockResolvedValue({ available: true, phase: 'active' }); + vi.mocked(api.listServers).mockResolvedValue([{ id: 'access-server-1', service_type: 'access-server', health: 'down' }]); + vi.stubGlobal('fetch', vi.fn(async () => new Response(JSON.stringify({ mode: 'node' })))); + render(); + await act(async () => {}); + expect(screen.getByRole('banner')).toHaveTextContent('Failed'); + vi.mocked(api.listServers).mockResolvedValue([{ id: 'access-server-1', service_type: 'access-server', health: 'up' }]); + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(screen.getByRole('banner')).toHaveTextContent('Healthy'); +}); it('waits for initial node authority before offering store actions', async () => { let resolve!: (status: api.NodeDeploymentStatus) => void; @@ -42,7 +60,8 @@ it('waits for initial node authority before offering store actions', async () => await act(async () => {}); expect(screen.queryByRole('button', { name: 'Add Store' })).not.toBeInTheDocument(); await act(async () => resolve({ available: true, phase: 'active' })); - fireEvent.click(screen.getByRole('button', { name: 'Add Store' })); + fireEvent.contextMenu(screen.getByRole('button', { name: /^datacenter$/ })); + fireEvent.click(screen.getByRole('menuitem', { name: 'Add Store' })); expect(screen.getByRole('dialog', { name: 'Add KV Store' })).toBeVisible(); }); @@ -54,4 +73,17 @@ it('shows a read-only console when the initial authority check fails', async () expect(screen.queryByText('Loading cluster authority…')).not.toBeInTheDocument(); expect(screen.getByRole('complementary', { name: 'Cluster tree sidebar' })).toBeVisible(); expect(screen.queryByRole('button', { name: 'Add Store' })).not.toBeInTheDocument(); + expect(screen.getByRole('alert')).toHaveTextContent('Group 0 unavailable · confirmed view is stale'); +}); + +it('automatically observes runtime capacity after selecting a Datacenter', async () => { + vi.mocked(api.listServers).mockResolvedValue([]); + vi.mocked(api.nodeDeploymentStatus).mockResolvedValue({ available: true, phase: 'active' }); + vi.stubGlobal('fetch', vi.fn(async () => new Response(JSON.stringify({ mode: 'node' })))); + render(); + await act(async () => {}); + expect(capacityRequests).toHaveBeenLastCalledWith(expect.objectContaining({ observeRuntime: false })); + fireEvent.click(screen.getByRole('button', { name: /^datacenter$/ })); + await act(async () => {}); + expect(capacityRequests).toHaveBeenLastCalledWith(expect.objectContaining({ observeRuntime: true, diskGroupId: undefined, diskId: undefined })); }); diff --git a/app/crowdb-web/ui/src/App.tsx b/app/crowdb-web/ui/src/App.tsx index 140127f87..cf02db29b 100644 --- a/app/crowdb-web/ui/src/App.tsx +++ b/app/crowdb-web/ui/src/App.tsx @@ -1,8 +1,6 @@ import { PanelDivider } from './components/PanelDivider'; -import { ClusterViewSwitch } from './shell/ClusterViewSwitch'; import { ClusterBreadcrumb } from './shell/ClusterBreadcrumb'; import { useClusterLayout } from './shell/useClusterLayout'; -import { CapacityObservations } from './shell/CapacityObservations'; // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. @@ -13,6 +11,7 @@ import { ToastProvider, useToast } from './contexts/ToastContext'; import { ActivityProvider, useActivity } from './contexts/ActivityContext'; import { useNodeAuthority } from './data/useNodeAuthority'; import { useClusterTree } from './data/useClusterTree'; +import { useServiceObservations } from './data/useServiceObservations'; import { useLogicalTree } from './data/useLogicalTree'; import { useCapacityTree } from './data/useCapacityTree'; import { MonitorSummary } from './managed/MonitorSummary'; @@ -26,6 +25,7 @@ import { ToastContainer } from './components/ToastContainer'; import { TreeNode } from './components/Tree'; import { ContextMenu, useContextMenu } from './components/ContextMenu'; import type { MenuTarget } from './topology/TopologyCanvas'; +import { CandidateProvider, useNodeCandidates } from './shell/NodeCandidatesContext'; import { AddRackDialog, AddNodeDialog, @@ -45,20 +45,15 @@ import { Domain } from './types'; import type { ConsoleDialogState } from './menus/context'; import { useClusterMenus } from './menus/useClusterMenus'; import { useCapacityMenus } from './menus/useCapacityMenus'; -import { setApiBase, resetCluster, compactDiskdbZones, rebuildDiskdbZoneBitmap, listServers } from './api'; +import { setApiBase, resetCluster, compactDiskdbZones, rebuildDiskdbZoneBitmap } from './api'; import { deployPortDefaultsForNode, diskdbPortDefaultsForNode, nextIdFromSuffix, nextNumericId } from './components/dialogs/defaults'; import { buildCrowdbKVServers, crowdbKvServerNodeIds } from './data/crowdbKvServers'; import { isCrowdbKVServerAvailable } from './data/crowdbKvServers'; import { toUiHealth } from './utils/entityDisplay'; import { ClusterView } from './views/ClusterView'; -import { ClusterOverview } from './storage/ClusterOverview'; +import { NodeStorage } from './storage/NodeStorage'; import { CreateFileDiskDialog } from './storage/CreateFileDiskDialog'; -import { useNavigationSnapshot } from './contexts/DomainContext'; -import { NodeWorkspace } from './storage/NodeWorkspace'; -import type { NodeSetupSummary } from './shell/nodeSetup'; import type { CandidateSummary } from './storage/types'; -import { InitializationProgress } from './shell/InitializationProgress'; -import { AutoJoin } from './shell/AutoJoin'; import { KvView } from './views/KvView'; import { CapacityView } from './views/CapacityView'; import { ChunkBrowser } from './chunk/ChunkBrowser'; @@ -88,17 +83,18 @@ export interface CrowdbConsoleProps { } function AppContent({ apiPrefix = '/api', readonly = false, modules, onEvent, managed = false, nodeManagement = false }: CrowdbConsoleProps) { - const { domain: activeDomain, clusterView, setClusterView, setDomain } = useDomain(); + const { domain: activeDomain, setDomain } = useDomain(); + const { snapshot: candidateSnapshot, candidates: candidateNodes } = useNodeCandidates(); const { selectedEntity, scopeEntity, scopeForDomain, selectionForDomain, selectEntity, clearSelection } = useSelection(); - const [diagram, setDiagram] = useState(false); const [addNodeRequest, setAddNodeRequest] = useState<{ rackId: number; nonce: number }>(); - useNavigationSnapshot(Domain.Cluster, 'cluster-diagram', () => { - const saved = diagram; return () => setDiagram(saved); - }); - const overviewScope = !scopeEntity || scopeEntity.type === 'Datacenter' || scopeEntity.type === 'Rack'; - const storageDetail = scopeEntity?.type === 'Disk' || scopeEntity?.type === 'DiskGroup'; - const domain = activeDomain === Domain.Cluster && (clusterView === 'capacity' || storageDetail) - ? Domain.Capacity : activeDomain; + const openNodeCandidates = useCallback((rackId: number) => setAddNodeRequest({ rackId, nonce: Date.now() }), []); + const domain = activeDomain; + const capacityActive = activeDomain === Domain.Cluster && !!scopeEntity + && ['Datacenter', 'Rack', 'Node', 'DiskGroup', 'Disk'].includes(scopeEntity.type) + && scopeEntity.section !== 'services'; + const topologyActive = activeDomain === Domain.Cluster + && !['DiskGroup', 'Disk'].includes(scopeEntity?.type ?? '') + && scopeEntity?.section !== 'storage' && scopeEntity?.section !== 'candidates'; const { success, error } = useToast(); const { log } = useActivity(); @@ -114,11 +110,6 @@ function AppContent({ apiPrefix = '/api', readonly = false, modules, onEvent, ma const [lastRefreshTime, setLastRefreshTime] = useState(new Date()); const [refreshing, setRefreshing] = useState(false); const [candidateSummaries, setCandidateSummaries] = useState>({}); - const [setupSummaries, setSetupSummaries] = useState>({}); - const observeSetup = useCallback((nodeId: number, summary: NodeSetupSummary) => { - setSetupSummaries(previous => Object.fromEntries(Object.entries({ ...previous, [nodeId]: summary }) - .sort((a, b) => b[1].observed_at_ms - a[1].observed_at_ms).slice(0, 64))); - }, []); const observeCandidates = useCallback((nodeId: number, summary: CandidateSummary) => { setCandidateSummaries(previous => { if (JSON.stringify(previous[nodeId]) === JSON.stringify(summary)) return previous; @@ -148,14 +139,13 @@ function AppContent({ apiPrefix = '/api', readonly = false, modules, onEvent, ma const { menuState, openMenu, closeMenu } = useContextMenu(); const nodeAuthority = useNodeAuthority(nodeManagement); - useEffect(() => { setCandidateSummaries({}); setSetupSummaries({}); }, [nodeAuthority?.cluster_id]); + useEffect(() => { setCandidateSummaries({}); }, [nodeAuthority?.cluster_id]); const managementAuthorized = !nodeManagement || nodeAuthority?.available === true; const draftEditable = nodeAuthority?.phase === 'unbound_draft'; const topologyReadonly = readonly || managed || (nodeManagement && !managementAuthorized && !draftEditable); const logicalReadonly = readonly || (nodeManagement && !managementAuthorized && !draftEditable); const ownsSidebar = domain === Domain.Dataset || domain === Domain.Iceberg || domain === Domain.S3 || (domain === Domain.Chunk || domain === Domain.ChunkKV); const physicalActive = domain === Domain.Cluster; - const capacityActive = domain === Domain.Capacity; const { racks, nodes, services: managedServers, nodeStores, nodeHealthById, nodeDiskGroups: clusterDiskGroups, loadNodeDisks, loadGroupDisks, loading: physLoading, error: physError, refresh: refreshPhysical } = useClusterTree({ enabled: true, managed, @@ -170,9 +160,9 @@ function AppContent({ apiPrefix = '/api', readonly = false, modules, onEvent, ma pollIntervalActive: 1000, pollIntervalInactive: 30000, }); - const { instances: diskdbInstances, usage: capacityUsage, hardwareCapacity, scanStatus: capacityScanStatus, loading: capLoading, error: capError, refresh: refreshCapacity, refreshRuntime: observeCapacityRuntime, observations: capacityObservations } = useCapacityTree({ + const { instances: diskdbInstances, usage: capacityUsage, hardwareCapacity, scanStatus: capacityScanStatus, loading: capLoading, error: capError, refresh: refreshCapacity, refreshRuntime: observeCapacityRuntime } = useCapacityTree({ enabled: domain === Domain.Capacity || domain === Domain.Cluster, - observeRuntime: capacityActive && (scopeEntity?.type === 'DiskGroup' || scopeEntity?.type === 'Disk'), + observeRuntime: capacityActive, diskGroupId: scopeForDomain(Domain.Cluster)?.parentIds?.disk_group_id !== undefined ? Number(scopeForDomain(Domain.Cluster)!.parentIds!.disk_group_id) : scopeForDomain(Domain.Cluster)?.type === 'DiskGroup' ? Number(scopeForDomain(Domain.Cluster)!.id) : undefined, @@ -184,7 +174,7 @@ function AppContent({ apiPrefix = '/api', readonly = false, modules, onEvent, ma const nodeDiskGroups = clusterDiskGroups; const diskSelection = scopeForDomain(Domain.Cluster); const clusterSelection = scopeForDomain(Domain.Cluster); - const workspaceNode = activeDomain === Domain.Cluster && nodeManagement && !managed && clusterSelection?.type === 'Node' + const workspaceNode = activeDomain === Domain.Cluster && nodeManagement && !managed && clusterSelection?.type === 'Node' && (clusterSelection.section === 'storage' || clusterSelection.section === 'candidates') ? nodes.find(node => String(node.id) === clusterSelection.id) : undefined; useEffect(() => { if (managed) return; @@ -208,32 +198,12 @@ function AppContent({ apiPrefix = '/api', readonly = false, modules, onEvent, ma ); const loading = physLoading || logLoading || capLoading; - const dataError = (domain === Domain.Cluster ? physError : domain === Domain.KV ? logError : domain === Domain.Capacity ? capError ?? physError : null); + const dataError = (domain === Domain.Cluster ? physError ?? (capacityActive ? capError : null) : domain === Domain.KV ? logError : null); const servers = useMemo(() => buildCrowdbKVServers(nodes, racks), [nodes, racks]); const serverNodeIds = useMemo(() => crowdbKvServerNodeIds(servers), [servers]); - const [standaloneServers, setAllServers] = useState([]); + const { servers: standaloneServers, refresh: refreshAllServers } = useServiceObservations( + !managed && (physicalActive || capacityActive || domain === Domain.Chunk || domain === Domain.ChunkKV), error); const allServers = managed ? managedServers : standaloneServers; - const serverErrorShownRef = useRef(false); - const refreshAllServers = useCallback(async () => { - try { - setAllServers(await listServers()); - serverErrorShownRef.current = false; - } catch (err) { - setAllServers([]); - // Only show the toast once per failure streak — polling retries - // every few seconds and would otherwise flood the UI with - // identical "backend unreachable" toasts. - if (!serverErrorShownRef.current) { - serverErrorShownRef.current = true; - error(`Failed to load server list: ${err instanceof Error ? err.message : 'backend unreachable'}`); - } - } - }, [error]); - useEffect(() => { - if (!managed && (physicalActive || capacityActive || domain === Domain.Chunk || domain === Domain.ChunkKV)) { - refreshAllServers(); - } - }, [managed, physicalActive, capacityActive, domain, diskdbInstances, refreshAllServers]); const diskdbNodeIds = useMemo( () => new Set([...allServers.filter((s) => s.service_type === 'diskdb' && s.node_id != null).map((s) => s.node_id!), ...nodes.filter(node => node.diskdb_server).map(node => node.id)]), [allServers, nodes], @@ -261,7 +231,7 @@ function AppContent({ apiPrefix = '/api', readonly = false, modules, onEvent, ma const servicePlans = useNodeServicePlans(stores, nodeDiskGroups, handleRefresh, !topologyReadonly && managementAuthorized); const clusterHealth: ClusterHealth = useMemo(() => { - if (dataError) return domain === Domain.Capacity ? 'Degraded' : 'Failed'; + if (dataError) return capacityActive && !physError ? 'Degraded' : 'Failed'; const statuses = [ ...groups.map((g) => toUiHealth(String((g as any).state || (g as any).health || ''))), ...nodes.map((node) => toUiHealth(String(node.kv_server?.health || ''))), @@ -275,7 +245,7 @@ function AppContent({ apiPrefix = '/api', readonly = false, modules, onEvent, ma if (planStates.some((state) => state === 'waiting' || state === 'deploying' || state === 'pending')) return 'Degraded'; if (statuses.every((status) => status === 'Healthy')) return 'Healthy'; return 'Unknown'; - }, [groups, nodes, allServers, servicePlans.plans, dataError, domain]); + }, [groups, nodes, allServers, servicePlans.plans, dataError, capacityActive, physError]); useEffect(() => { if (managed) return; for (const [id, plan] of Object.entries(servicePlans.plans)) { @@ -290,7 +260,7 @@ function AppContent({ apiPrefix = '/api', readonly = false, modules, onEvent, ma // After cluster init succeeds, refresh the tree so the system group // appears. Init only bootstraps store 0 / group 0; store creation is - // a separate step the user initiates via the "+" button. + // a separate step in the Datacenter context menu. const handleInitSuccess = useCallback(async () => { await handleRefresh(); setDialog((d) => ({ ...d, initCluster: false })); @@ -343,7 +313,7 @@ function AppContent({ apiPrefix = '/api', readonly = false, modules, onEvent, ma if (node) selectEntity({ domain: Domain.Cluster, type: 'Node', id: String(node.id), parentIds: { rack_id: node.rack_id }, section: 'storage' }); }, [nodes, selectEntity]); const manageNodeStorage = nodeManagement && !managed ? openNodeStorage : undefined; - const menuContext = { readonly, managed, nodeManagement, onManageNodeStorage: manageNodeStorage, managementAuthorized, domain: activeDomain, physicalActive: activeDomain === Domain.Cluster, modules, requestDelete, runMutation, serverNodeIds, diskdbNodeIds, allServers, setDialog, capacityUsage }; + const menuContext = { readonly, managed, nodeManagement, onAddNodeCandidate: openNodeCandidates, onManageNodeStorage: manageNodeStorage, managementAuthorized, draftEditable, clusterInitialized, domain: activeDomain, physicalActive: activeDomain === Domain.Cluster, modules, requestDelete, runMutation, serverNodeIds, diskdbNodeIds, allServers, setDialog, capacityUsage }; const buildMenuItems = useClusterMenus(menuContext); const buildCapacityMenuItems = useCapacityMenus(menuContext); @@ -372,9 +342,8 @@ function AppContent({ apiPrefix = '/api', readonly = false, modules, onEvent, ma ); const onTreeNodeClick = useCallback((node: TreeNode) => { - if (node.section) setClusterView(node.section === 'services' ? 'topology' : 'capacity'); setCanvasFocusRequest({ targetId: node.section ? `N-${node.rawId}` : node.id, subtree: true, nonce: Date.now() }); - }, [setClusterView]); + }, []); const handleAdd = useCallback(() => { if (readonly || (nodeManagement && !managementAuthorized && !draftEditable)) return; @@ -558,6 +527,7 @@ function AppContent({ apiPrefix = '/api', readonly = false, modules, onEvent, ma diskdbInstanceIdByNodeId={diskdbInstanceIdByNodeId} servicePlans={servicePlans.plans} candidateSummaries={candidateSummaries} + candidateHasNodes={!candidateSnapshot || candidateNodes.length > 0} /> {!ownsSidebar && !clusterLayout.hidden && !clusterLayout.overlayNavigation && - {dataError &&
- {domain === Domain.Capacity ? `${dataError.message} — retrying` : 'Backend unreachable — retrying'} + {(dataError || nodeAuthority?.phase === 'authority_unavailable') &&
+ {nodeAuthority?.phase === 'authority_unavailable' ? 'Group 0 unavailable · confirmed view is stale' + : capacityActive ? `${dataError!.message} — retrying` : 'Backend unreachable — retrying'}
} - {activeDomain === Domain.Cluster && <> - - - - {!storageDetail && } - {overviewScope && domain === Domain.Cluster &&
- + {activeDomain === Domain.Cluster && } + {kvEnabled && ( )} - @@ -665,7 +590,7 @@ function AppContent({ apiPrefix = '/api', readonly = false, modules, onEvent, ma {!ownsSidebar && - + } {selectedEntity && !ownsSidebar && !clusterLayout.drawer && + + diff --git a/app/crowdb-web/ui/src/access/PrefixList.tsx b/app/crowdb-web/ui/src/access/PrefixList.tsx new file mode 100644 index 000000000..d5bc28009 --- /dev/null +++ b/app/crowdb-web/ui/src/access/PrefixList.tsx @@ -0,0 +1,13 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. +import { buttonClass, inputClass } from './Workbench'; + +export function PrefixList({ resource, value, change, submit, busy }: { + resource: 'Bucket' | 'Object'; value: string; change: (value: string) => void; + submit: () => void; busy: boolean; +}) { + return
{ event.preventDefault(); submit(); }}> + + +
; +} diff --git a/app/crowdb-web/ui/src/dataset/import/digest.ts b/app/crowdb-web/ui/src/access/digest.ts similarity index 100% rename from app/crowdb-web/ui/src/dataset/import/digest.ts rename to app/crowdb-web/ui/src/access/digest.ts diff --git a/app/crowdb-web/ui/src/access/native.ts b/app/crowdb-web/ui/src/access/native.ts index 170491261..3d2f1b2fd 100644 --- a/app/crowdb-web/ui/src/access/native.ts +++ b/app/crowdb-web/ui/src/access/native.ts @@ -15,23 +15,32 @@ export async function readJson(response: Response): Promise { await check(response); return response.status === 204 ? undefined as T : response.json(); } +export class AccessError extends Error { + constructor(readonly status: number, message: string, readonly errorType?: string) { + super(`HTTP ${status}: ${message}`); + } + get unknownOutcome() { return this.errorType === 'CommitStateUnknownException' || this.status >= 500; } +} export async function check(response: Response): Promise { if (response.ok) return; const text = new TextDecoder().decode(await previewBytes(response)); let message = text; + let errorType: string | undefined; try { const body = JSON.parse(text); message = body.error?.message ?? body.error ?? body.message ?? text; + errorType = body.error?.type; } catch { const xml = new DOMParser().parseFromString(text, 'application/xml'); message = xml.querySelector('Message')?.textContent ?? text; + errorType = xml.querySelector('Code')?.textContent ?? undefined; } - throw new Error(`HTTP ${response.status}: ${message || response.statusText}`); + throw new AccessError(response.status, message || response.statusText, errorType); } -export async function iceberg(path: string, token: string, method = 'GET', body?: object, signal?: AbortSignal): Promise { +export async function iceberg(path: string, token: string, method = 'GET', body?: object | string, signal?: AbortSignal, requestKey?: string): Promise { const response = await fetch(`${getApiBase()}/access/iceberg${path}`, { - method, signal, headers: { ...(token ? { Authorization: `Bearer ${token}` } : {}), ...(body ? { 'Content-Type': 'application/json' } : {}) }, - ...(body ? { body: JSON.stringify(body) } : {}), + method, signal, headers: { ...(token ? { Authorization: `Bearer ${token}` } : {}), ...(body ? { 'Content-Type': 'application/json' } : {}), ...(requestKey ? { 'Idempotency-Key': requestKey } : {}) }, + ...(body ? { body: typeof body === 'string' ? body : JSON.stringify(body) } : {}), }); await check(response); return response.status === 204 ? undefined : parseIcebergJson(await boundedText(response)); diff --git a/app/crowdb-web/ui/src/access/resourceMenus.test.tsx b/app/crowdb-web/ui/src/access/resourceMenus.test.tsx new file mode 100644 index 000000000..86753de81 --- /dev/null +++ b/app/crowdb-web/ui/src/access/resourceMenus.test.tsx @@ -0,0 +1,110 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { useIcebergMenu } from '../iceberg/ResourceMenu'; +import { useS3Menu } from '../s3/ResourceMenu'; +import { iceberg, s3 } from './native'; +import { uploadObject } from './s3Transfer'; +vi.mock('./native', () => ({ iceberg: vi.fn(async () => ({})), s3: vi.fn(async () => new Response()), objectPath: (bucket: string, key?: string) => `/${bucket}${key === undefined ? '' : '/' + key}`, previewBytes: vi.fn() })); +vi.mock('./s3Transfer', () => ({ uploadObject: vi.fn(), downloadObject: vi.fn() })); +beforeEach(() => { vi.stubGlobal('crypto', { getRandomValues: crypto.getRandomValues.bind(crypto) }); }); +afterEach(() => { vi.clearAllMocks(); vi.unstubAllGlobals(); }); +function Catalog({ namespace = 'first', readonly = false }) { + const menu = useIcebergMenu(readonly, () => true, async () => {}); + return <>{menu.overlay}; +} +function NamespaceMenu() { + const menu = useIcebergMenu(false, () => true, async () => {}); + return <>{menu.overlay}; +} +it('pins the exact Iceberg target across selection changes and requires confirmation', async () => { + const { rerender } = render(); + fireEvent.contextMenu(screen.getByRole('button', { name: 'Resource' })); + fireEvent.click(screen.getByRole('menuitem', { name: 'Drop table…' })); + expect(iceberg).not.toHaveBeenCalled(); + rerender(); + expect(screen.getByRole('dialog')).toHaveTextContent('first / table'); + fireEvent.click(screen.getByRole('button', { name: 'Drop table' })); + await waitFor(() => expect(iceberg).toHaveBeenCalledWith('/v1/namespaces/first/tables/table', '', 'DELETE')); +}); +it('keeps readonly Iceberg mutations disabled', () => { + render(); fireEvent.contextMenu(screen.getByRole('button', { name: 'Resource' })); + for (const item of screen.getAllByRole('menuitem')) expect(item).toBeDisabled(); + expect(iceberg).not.toHaveBeenCalled(); +}); +it('prefills a valid table name in the Iceberg table editor', async () => { + render(); fireEvent.contextMenu(screen.getByRole('button', { name: 'Namespace' })); + fireEvent.click(screen.getByRole('menuitem', { name: 'Create table…' })); + await waitFor(() => expect(screen.getByRole('textbox', { name: 'Table name' })).toBeVisible()); + expect((screen.getByRole('textbox', { name: 'Table name' }) as HTMLInputElement).value).toMatch(/^table-[0-9a-f]{8}$/); +}); +it('prefills a valid namespace name for one-click creation', async () => { + render(<>); fireEvent.contextMenu(screen.getByRole('button', { name: 'Namespace' })); + fireEvent.click(screen.getByRole('menuitem', { name: 'Create namespace…' })); + const input = screen.getByRole('textbox', { name: 'Namespace name' }); + expect((input as HTMLInputElement).value).toMatch(/^namespace-[0-9a-f]{8}$/); + fireEvent.click(screen.getByRole('button', { name: 'Create namespace' })); + await waitFor(() => expect(iceberg).toHaveBeenCalledWith('/v1/namespaces', '', 'POST', expect.objectContaining({ namespace: ['first', expect.stringMatching(/^namespace-[0-9a-f]{8}$/)] }))); +}); +function ObjectMenu({ readonly = false }) { + const menu = useS3Menu(readonly, true, async () => {}, () => {}, () => {}); + return <>{menu.overlay}; +} +it('offers explicit object reads in readonly mode without permitting mutations', () => { + render(); fireEvent.contextMenu(screen.getByRole('button', { name: 'Object' })); + expect(screen.getByRole('menuitem', { name: 'Preview first 4 KiB' })).toBeEnabled(); + expect(screen.getByRole('menuitem', { name: 'Download' })).toBeEnabled(); + expect(screen.getByRole('menuitem', { name: 'Replace object…' })).toBeDisabled(); + expect(screen.getByRole('menuitem', { name: 'Delete object…' })).toBeDisabled(); + expect(s3).not.toHaveBeenCalled(); +}); +it('deletes only the confirmed S3 Object', async () => { + render(); fireEvent.contextMenu(screen.getByRole('button', { name: 'Object' })); + fireEvent.click(screen.getByRole('menuitem', { name: 'Delete object…' })); + expect(screen.getByRole('dialog')).toHaveTextContent('photos / a/b'); + expect(s3).not.toHaveBeenCalled(); + fireEvent.click(screen.getByRole('button', { name: 'Delete object' })); + await waitFor(() => expect(s3).toHaveBeenCalledWith('DELETE', '/photos/a/b')); +}); +function BucketMenu() { + const menu = useS3Menu(false, true, async () => {}, () => {}, () => {}); + return <>{menu.overlay}; +} +function S3RootMenu() { + const menu = useS3Menu(false, true, async () => {}, () => {}, () => {}); + return <>{menu.overlay}; +} +it('prefills a valid bucket name for one-click creation', async () => { + render(); fireEvent.contextMenu(screen.getByRole('button', { name: 'S3' })); + fireEvent.click(screen.getByRole('menuitem', { name: 'Create bucket…' })); + const input = screen.getByRole('textbox', { name: 'New bucket' }); + expect((input as HTMLInputElement).value).toMatch(/^bucket-[0-9a-f]{8}$/); + fireEvent.click(screen.getByRole('button', { name: 'Create bucket' })); + await waitFor(() => expect(s3).toHaveBeenCalledWith('PUT', expect.stringMatching(/^\/bucket-[0-9a-f]{8}$/))); +}); +it('uploads a batch using each filename as the object key', async () => { + render(); fireEvent.contextMenu(screen.getByRole('button', { name: 'Bucket' })); + fireEvent.click(screen.getByRole('menuitem', { name: 'Upload object…' })); + const first = new File(['a'], 'a.txt'); const second = new File(['b'], 'b.txt'); + fireEvent.change(screen.getByLabelText('Object files'), { target: { files: [first, second] } }); + expect(screen.queryByLabelText('Object key')).toBeNull(); + fireEvent.click(screen.getByRole('button', { name: 'Upload object' })); + await waitFor(() => expect(uploadObject).toHaveBeenCalledTimes(2)); + expect(uploadObject).toHaveBeenNthCalledWith(1, 'photos', 'a.txt', first, expect.any(AbortSignal), expect.any(Function)); + expect(uploadObject).toHaveBeenNthCalledWith(2, 'photos', 'b.txt', second, expect.any(AbortSignal), expect.any(Function)); +}); +it('reports each batch outcome and continues after an individual upload fails', async () => { + vi.mocked(uploadObject).mockResolvedValueOnce(undefined).mockRejectedValueOnce(new Error('upload rejected')).mockRejectedValueOnce(new DOMException('stopped', 'AbortError')); + render(); fireEvent.contextMenu(screen.getByRole('button', { name: 'Bucket' })); + fireEvent.click(screen.getByRole('menuitem', { name: 'Upload object…' })); + fireEvent.change(screen.getByLabelText('Object files'), { target: { files: [new File(['a'], 'a.txt'), new File(['b'], 'b.txt'), new File(['c'], 'c.txt')] } }); + fireEvent.click(screen.getByRole('button', { name: 'Upload object' })); + const results = await screen.findByRole('list', { name: 'Object upload results' }); + await waitFor(() => { + expect(results).toHaveTextContent('a.txt: success'); + expect(results).toHaveTextContent('b.txt: failed'); + expect(results).toHaveTextContent('c.txt: unknown'); + }); + expect(uploadObject).toHaveBeenCalledTimes(3); +}); diff --git a/app/crowdb-web/ui/src/api.ts b/app/crowdb-web/ui/src/api.ts index ca463ff53..d9bb422fa 100644 --- a/app/crowdb-web/ui/src/api.ts +++ b/app/crowdb-web/ui/src/api.ts @@ -643,6 +643,7 @@ export async function clusterInitialization(options?: RequestOptions): Promise; enabled: boolean; target_cluster_id: string | null; mode: 'standard'; @@ -660,10 +661,10 @@ export async function nodeAutoJoin(options?: RequestOptions): Promise { +export async function setNodeAutoJoin(enabled: boolean, target?: 'local-draft', discoveryId?: string): Promise { return jsonOrThrow(await fetchWithOptions('/api/node/auto-join', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ enabled, target }), skipDeduplication: true, + body: JSON.stringify({ enabled, target, ...(discoveryId ? { discovery_id: discoveryId } : {}) }), skipDeduplication: true, })); } diff --git a/app/crowdb-web/ui/src/chunk/ChunkBrowser.tsx b/app/crowdb-web/ui/src/chunk/ChunkBrowser.tsx index bc86d14c7..5f7f20a53 100644 --- a/app/crowdb-web/ui/src/chunk/ChunkBrowser.tsx +++ b/app/crowdb-web/ui/src/chunk/ChunkBrowser.tsx @@ -247,7 +247,7 @@ export function ChunkBrowser({ active, onPlacement, openRequest, racks, nodes, s {!isPkvSelection && page &&

{page.owners} owners · scanned {page.scanned} records this page · observed {new Date(page.observed_at_ms).toLocaleTimeString()}

} {!isPkvSelection && page?.owners === 0 &&

No live ChunkDB service is registered. Deploy ChunkDB in Cluster before browsing chunks.

} {!isPkvSelection && !!page?.failures.length &&
Partial result. Retry this scope after owner recovery; no continuation is issued while an owner is missing.
} - + {!!rows.length && !visibleRows.length &&

No matches in this window. Clear the filter or continue to the next window.

} {!isPkvSelection && page && !rows.length && !busy &&

No matching chunks in this scan window.{page.next && ' Continue scanning to find later matches.'}

} {!isPkvSelection && page &&