From 769c163c507fda94d8cce46ca7427ae6efff9d93 Mon Sep 17 00:00:00 2001 From: aaltshuler Date: Mon, 26 Oct 2026 11:16:04 +0200 Subject: [PATCH 1/4] feat(maintenance)!: add served optimization and interrupted-operation recovery --- changelog.d/served-optimize-cli.added.md | 9 + changelog.d/served-optimize.breaking.md | 19 + crates/omnigraph-api-types/src/lib.rs | 15 +- crates/omnigraph-api-types/src/operations.rs | 34 +- crates/omnigraph-catalog/src/lib.rs | 2 +- .../src/schema_publication.rs | 20 +- crates/omnigraph-cli/src/app.rs | 1 + crates/omnigraph-cli/src/cli.rs | 3 +- crates/omnigraph-cli/src/cluster_remote.rs | 2 +- .../src/commands/maintenance/args.rs | 15 +- .../src/commands/maintenance/mod.rs | 121 +- .../src/commands/maintenance/remote.rs | 469 + crates/omnigraph-cli/src/planes.rs | 36 +- .../tests/integration/cli_cluster.rs | 396 + .../tests/integration/cli_data.rs | 20 +- .../tests/integration/system_remote.rs | 123 + .../tests/support/managed_http.rs | 44 + crates/omnigraph-cluster/src/admission.rs | 61 +- crates/omnigraph-cluster/src/authorization.rs | 2 +- .../src/deployment/bootstrap_fault_tests.rs | 404 +- .../src/deployment/execution.rs | 1 + crates/omnigraph-cluster/src/lib.rs | 5 +- crates/omnigraph-cluster/src/maintenance.rs | 1930 ++++ crates/omnigraph-cluster/src/store.rs | 46 + crates/omnigraph-dst/cost_table.txt | 3 + crates/omnigraph-dst/tests/scenarios.rs | 67 + crates/omnigraph-policy/src/lib.rs | 59 +- crates/omnigraph-server/src/deployment.rs | 2 +- .../omnigraph-server/src/http_operations.rs | 10 +- crates/omnigraph-server/src/ingress.rs | 63 +- crates/omnigraph-server/src/lib.rs | 24 +- crates/omnigraph-server/src/maintenance.rs | 1079 ++ .../omnigraph-server/tests/boot_settings.rs | 791 +- .../tests/integration/openapi.rs | 203 + crates/omnigraph/src/db/mod.rs | 16 +- crates/omnigraph/src/db/omnigraph.rs | 88 +- crates/omnigraph/src/db/omnigraph/optimize.rs | 862 +- .../src/db/omnigraph/optimize_recovery.rs | 260 + .../src/db/omnigraph/schema_apply/prepared.rs | 2 +- .../omnigraph/src/db/omnigraph/table_ops.rs | 23 +- crates/omnigraph/src/table_store.rs | 225 +- crates/omnigraph/tests/failpoints.rs | 66 +- .../tests/integration/maintenance.rs | 517 +- .../omnigraph/tests/policy_engine_chassis.rs | 22 +- docs/dev/architecture.md | 3 + docs/dev/control-plane.md | 16 + docs/dev/http-api.md | 4 +- docs/dev/testing.md | 15 + docs/rfcs/2026-10-25-served-optimization.md | 229 + docs/rfcs/README.md | 1 + docs/user/cli/reference.md | 2 +- docs/user/cli/targets.md | 6 +- docs/user/operations/maintenance.md | 176 +- docs/user/operations/policy.md | 11 + docs/user/operations/storage-failures.md | 21 + openapi.json | 8672 ++++++++++------- skills/omnigraph/SKILL.md | 2 +- skills/omnigraph/references/commands.md | 16 + 58 files changed, 13611 insertions(+), 3723 deletions(-) create mode 100644 changelog.d/served-optimize-cli.added.md create mode 100644 changelog.d/served-optimize.breaking.md create mode 100644 crates/omnigraph-cli/src/commands/maintenance/remote.rs create mode 100644 crates/omnigraph-cluster/src/maintenance.rs create mode 100644 crates/omnigraph-server/src/maintenance.rs create mode 100644 crates/omnigraph/src/db/omnigraph/optimize_recovery.rs create mode 100644 docs/rfcs/2026-10-25-served-optimization.md diff --git a/changelog.d/served-optimize-cli.added.md b/changelog.d/served-optimize-cli.added.md new file mode 100644 index 000000000..97e47bcc2 --- /dev/null +++ b/changelog.d/served-optimize-cli.added.md @@ -0,0 +1,9 @@ +- `omnigraph optimize` supports registered server targets with an explicitly + selected graph. `--no-wait` returns after acceptance, `--operation-id` observes + an existing operation without resubmission, and `--timeout` bounds observation + without cancelling server work. The CLI retains the original operation ID + through uncertain delivery; direct-storage optimize keeps its existing output. + Administrators can reconcile an interrupted original ID with + `--writers-stopped` on an explicit direct-root target after writer and + accepted-I/O exclusion. This never runs another optimization or unlocks a + held owner. diff --git a/changelog.d/served-optimize.breaking.md b/changelog.d/served-optimize.breaking.md new file mode 100644 index 000000000..db1de208c --- /dev/null +++ b/changelog.d/served-optimize.breaking.md @@ -0,0 +1,19 @@ +- `optimize` now checks the installed graph policy's explicit `optimize` action +on target branch `main`, including direct engine and CLI access. Existing +policies must grant this action before those actors can optimize. Graphs +without an installed policy retain direct storage behavior. + +- Cluster servers can opt into bounded, authenticated optimization with durable +original-ID receipts. It runs under the existing native owner, serializes with +configuration deployment and preserves unresolved work without replay. +Source inventory ceilings do not replace process CPU, memory or I/O limits. +The complete result envelope is reserved before effects. Original-ID recovery +records interrupted publication under stopped-writer exclusion without replay. +Root-bound signed-token/OIDC servers refuse new optimization until resource +admission is supported; direct HTTP cannot bypass that gate. + +- Vector index reconciliation now propagates statistics, storage and builder +failures instead of reporting them as pending training data. An existing index +is deferred only when it has no non-null vectors; Lance chooses the supported +training layout for positive input. +Served source preflight also refuses unsupported index formats before rewriting. diff --git a/crates/omnigraph-api-types/src/lib.rs b/crates/omnigraph-api-types/src/lib.rs index f07f23307..3b2013b1e 100644 --- a/crates/omnigraph-api-types/src/lib.rs +++ b/crates/omnigraph-api-types/src/lib.rs @@ -2013,6 +2013,10 @@ fn access_slot(action: omnigraph_policy::PolicyAction) -> Option { P::InvokeQuery => Some(AccessSlot::GraphWide), // Server- or operator-scoped: no graph's access report names them. P::Admin | P::GraphList | P::ConfigManage => None, + // This report has a closed wire vocabulary consumed by older clients. + // Maintenance status checks Optimize separately on main; do not add a + // new AccessAction without deliberately evolving the report contract. + P::Optimize => None, } } @@ -2036,7 +2040,8 @@ impl From for omnigraph_policy::PolicyAction { /// What the caller's own credential may do on one graph, by the graph's /// installed policy. It names actions, never the rules that allow them, and -/// says nothing about any other actor. +/// says nothing about any other actor. This closed vocabulary omits optimize; +/// maintenance status checks that permission separately on main. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)] pub struct AccessOutput { /// The actor the credential resolves to. Absent only on a server that runs @@ -3593,6 +3598,14 @@ mod tests { transitions, [AccessAction::BranchCreate, AccessAction::BranchMerge] ); + assert_eq!(access_slot(omnigraph_policy::PolicyAction::Optimize), None); + assert!( + AccessAction::ALL + .into_iter() + .all(|action| omnigraph_policy::PolicyAction::from(action) + != omnigraph_policy::PolicyAction::Optimize), + "the existing closed access vocabulary must stay readable by older clients" + ); } #[test] diff --git a/crates/omnigraph-api-types/src/operations.rs b/crates/omnigraph-api-types/src/operations.rs index 9a4aa46dd..fa48612c9 100644 --- a/crates/omnigraph-api-types/src/operations.rs +++ b/crates/omnigraph-api-types/src/operations.rs @@ -210,6 +210,39 @@ pub const HTTP_OPERATIONS: &[HttpOperation] = &[ Surface::absent(DEPLOYMENT), Surface::bound(&["cluster plan"]), ), + operation( + "maintenance status", + HttpMethod::Get, + "/cluster/maintenance", + Capability::Read, + None, + Surface::absent("Maintenance is not an MCP operation."), + Surface::restricted( + &["optimize"], + "Read native admission and resource ceilings before submission.", + ), + ), + operation( + "optimize", + HttpMethod::Post, + "/cluster/maintenance", + Capability::Write, + None, + Surface::absent("Maintenance is not an MCP operation."), + Surface::bound(&["optimize"]), + ), + operation( + "maintenance lookup", + HttpMethod::Get, + "/cluster/maintenance/{operation_id}", + Capability::Read, + None, + Surface::absent("Maintenance is not an MCP operation."), + Surface::restricted( + &["optimize"], + "--operation-id observes the original invocation without execution.", + ), + ), operation( "list graphs", HttpMethod::Get, @@ -621,7 +654,6 @@ pub const CLI_ONLY_OPERATIONS: &[CliOnlyOperation] = &[ cli_only("schema upgrade-system-columns", STORAGE), cli_only("init", STORAGE), cli_only("upgrade", STORAGE), - cli_only("optimize", STORAGE), cli_only("rebuild-full-text-indexes", STORAGE), cli_only("repair", STORAGE), cli_only("cleanup", STORAGE), diff --git a/crates/omnigraph-catalog/src/lib.rs b/crates/omnigraph-catalog/src/lib.rs index 8523d4e80..f23c0d510 100644 --- a/crates/omnigraph-catalog/src/lib.rs +++ b/crates/omnigraph-catalog/src/lib.rs @@ -22,7 +22,7 @@ mod row; mod schema_publication; pub use schema_publication::{ SchemaPublicationCandidate, SchemaPublicationEvidence, read_schema_publication_at, - read_schema_publication_candidate_at, + read_schema_publication_candidate_at, read_schema_publication_candidate_at_with_session, }; use std::collections::{HashMap, HashSet}; diff --git a/crates/omnigraph-catalog/src/schema_publication.rs b/crates/omnigraph-catalog/src/schema_publication.rs index a61b0e1fa..6973371b3 100644 --- a/crates/omnigraph-catalog/src/schema_publication.rs +++ b/crates/omnigraph-catalog/src/schema_publication.rs @@ -43,6 +43,24 @@ pub async fn read_schema_publication_candidate_at( version: u64, original_id: Option<&str>, settlement_id: Option<&str>, +) -> Result> { + read_schema_publication_candidate_at_with_session( + root_uri, + version, + original_id, + settlement_id, + None, + ) + .await +} + +/// The same exact-version proof, retaining an already captured provider. +pub async fn read_schema_publication_candidate_at_with_session( + root_uri: &str, + version: u64, + original_id: Option<&str>, + settlement_id: Option<&str>, + session: Option<&std::sync::Arc>, ) -> Result> { if version == 0 || original_id @@ -58,7 +76,7 @@ pub async fn read_schema_publication_candidate_at( let dataset = match open_dataset( &manifest_uri(root_uri), VersionResolution::At(version), - None, + session, crate::instrumentation::manifest_wrapper(), ) .await diff --git a/crates/omnigraph-cli/src/app.rs b/crates/omnigraph-cli/src/app.rs index e67af2d6e..f52b21812 100644 --- a/crates/omnigraph-cli/src/app.rs +++ b/crates/omnigraph-cli/src/app.rs @@ -201,6 +201,7 @@ fn finish( if error .downcast_ref::() .is_some() + || error.is::() { stdout.flush()?; return Ok(Completion::exit(5)); diff --git a/crates/omnigraph-cli/src/cli.rs b/crates/omnigraph-cli/src/cli.rs index 224c484d5..b5e822111 100644 --- a/crates/omnigraph-cli/src/cli.rs +++ b/crates/omnigraph-cli/src/cli.rs @@ -47,6 +47,7 @@ pub(crate) struct Cli { /// On a served write, and on every request `mcp` relays, it is an assertion: /// the request carries `omnigraph-if-actor`, and the server refuses it /// before anything runs when the bearer token resolves to another actor. + /// Served optimize derives its actor only from the credential and refuses --as. /// With a policy configured but no actor set, a direct operation is /// denied — see docs/user/operations/policy.md. #[arg(long = "as", global = true, value_name = "ACTOR", value_parser = crate::invocation::single_selector("--as"))] @@ -145,7 +146,7 @@ pub(crate) enum Command { /// Convert a standalone graph's storage offline from format v8, v9 or v13 to /// the current format this binary serves (v16); other formats are rebuilt by export and load Upgrade(UpgradeArgs), - /// Compact small Lance fragments in every backing dataset of the graph + /// Compact small Lance fragments in every backing dataset of the graph on main Optimize(OptimizeArgs), /// Rebuild all full-text indexes from one branch's current rows /// diff --git a/crates/omnigraph-cli/src/cluster_remote.rs b/crates/omnigraph-cli/src/cluster_remote.rs index f6668297b..c7c78cbfb 100644 --- a/crates/omnigraph-cli/src/cluster_remote.rs +++ b/crates/omnigraph-cli/src/cluster_remote.rs @@ -417,7 +417,7 @@ fn is_wait_timeout(error: &color_eyre::Report, deadline: Instant) -> bool { error.is::() || Instant::now() >= deadline } -fn retry_observation(error: &color_eyre::Report) -> bool { +pub(crate) fn retry_observation(error: &color_eyre::Report) -> bool { error.is::() || error.downcast_ref::().is_some_and(|error| { error.is_timeout() diff --git a/crates/omnigraph-cli/src/commands/maintenance/args.rs b/crates/omnigraph-cli/src/commands/maintenance/args.rs index f7f4009d5..6b8ad7826 100644 --- a/crates/omnigraph-cli/src/commands/maintenance/args.rs +++ b/crates/omnigraph-cli/src/commands/maintenance/args.rs @@ -1,4 +1,4 @@ -//! Arguments for direct graph maintenance. +//! Arguments for graph maintenance. use clap::Args; @@ -16,6 +16,19 @@ pub(crate) struct UpgradeArgs { #[derive(Debug, Args)] pub(crate) struct OptimizeArgs { + /// Return after the server accepts the operation; work continues remotely + #[arg(long)] + pub(crate) no_wait: bool, + /// Observe an existing operation; never submit another optimization + #[arg(long, value_name = "ID")] + pub(crate) operation_id: Option, + /// Reconcile this original operation on an explicit direct root after + /// excluding prior writers and accepted graph/control I/O + #[arg(long, requires = "operation_id", conflicts_with_all = ["no_wait", "timeout"])] + pub(crate) writers_stopped: bool, + /// Maximum seconds to observe server work (default 300); never cancels work + #[arg(long, value_parser = clap::value_parser!(u64).range(1..=3600))] + pub(crate) timeout: Option, #[arg(long)] pub(crate) json: bool, } diff --git a/crates/omnigraph-cli/src/commands/maintenance/mod.rs b/crates/omnigraph-cli/src/commands/maintenance/mod.rs index a6557054c..ad32d80c8 100644 --- a/crates/omnigraph-cli/src/commands/maintenance/mod.rs +++ b/crates/omnigraph-cli/src/commands/maintenance/mod.rs @@ -1,4 +1,4 @@ -//! Direct graph maintenance through the existing admission and engine owners. +//! Graph maintenance through the existing admission and engine owners. use std::io::Write; @@ -10,6 +10,7 @@ use crate::helpers::{confirm_destructive, parse_duration_arg, resolve_actor}; use crate::output::{write_cleanup, write_full_text_rebuild, write_optimize, write_repair}; pub(crate) mod args; +pub(crate) mod remote; pub(crate) use args::{ CleanupArgs, OptimizeArgs, RebuildFullTextIndexesArgs, RepairArgs, UpgradeArgs, }; @@ -39,7 +40,16 @@ pub(crate) async fn run_optimize( stdout: &mut dyn Write, stderr: &mut dyn Write, ) -> Result { - let OptimizeArgs { json } = args; + if args.writers_stopped { + return recover_optimize(args, scope, stdout, stderr).await; + } + if matches!( + scope.target.binding, + crate::targets::TargetBinding::Server { .. } + ) { + return remote::run(args, scope, stdout, stderr).await; + } + let OptimizeArgs { json, .. } = args; let uri = crate::scope::direct_graph_uri(scope.target, scope.graph).await?; let binding = crate::scope::capture_storage_binding(scope.target)?.for_uri(&uri)?; crate::admission::ensure_graph_with_binding(&binding, stderr).await?; @@ -48,11 +58,116 @@ pub(crate) async fn run_optimize( omnigraph::storage::storage_for_binding(&binding)?, ) .await?; - let stats = db.optimize().await?; + let actor = resolve_actor(scope.operator, scope.actor, stderr)?; + let stats = db.optimize_as(actor.as_deref()).await?; write_optimize(&uri, &stats, json, stdout)?; Ok(Completion::success()) } +async fn recover_optimize( + args: OptimizeArgs, + scope: MaintenanceScope<'_>, + stdout: &mut dyn Write, + stderr: &mut dyn Write, +) -> Result { + use color_eyre::eyre::{WrapErr, eyre}; + use omnigraph_cluster::{ + ClusterAdmissionPurpose, IdentityAuthorization, MaintenanceCompletion, + MaintenancePublication, + }; + if !scope.target.is_explicit() + || !matches!( + scope.target.binding, + crate::targets::TargetBinding::DirectRoot { .. } + ) + { + bail!("optimization recovery requires an explicit direct-root --target"); + } + let id = args + .operation_id + .as_deref() + .ok_or_else(|| eyre!("recovery requires --operation-id"))?; + let graph = crate::scope::graph_selection(scope.target, scope.graph)? + .ok_or_else(|| eyre!("optimization recovery requires a graph"))?; + let actor = resolve_actor(scope.operator, scope.actor, stderr)? + .ok_or_else(|| eyre!("optimization recovery requires an explicit or configured actor"))?; + let diagnostic = |error: omnigraph_cluster::Diagnostic| { + eyre!("[{}] {}: {}", error.code, error.path, error.message) + }; + let identity = IdentityAuthorization::authenticated(actor).map_err(diagnostic)?; + let binding = crate::scope::capture_storage_binding(scope.target)?; + let admission = omnigraph_cluster::acquire_cluster_admission_with_binding( + &binding, + ClusterAdmissionPurpose::MaintenanceRecovery { + operation_id: id.to_owned(), + }, + ) + .await + .map_err(diagnostic)? + .ok_or_else(|| eyre!("optimization recovery requires an initialized cluster"))?; + let custody = format!( + "Operation-ID: {id}; cluster-root: {}; lock-ID: {}", + omnigraph::storage::redacted_storage_uri(admission.canonical_root()), + admission.lock_id(), + ); + writeln!(stderr, "{custody}") + .and_then(|()| stderr.flush()) + .wrap_err(custody.clone())?; + let result = omnigraph_cluster::reconcile_maintenance(&admission, graph, id, &identity) + .await + .map_err(diagnostic) + .wrap_err(custody.clone())?; + let delivery = (|| -> Result<()> { + if args.json { + crate::output::write_json( + &serde_json::json!({ + "maintenance": {"status": "complete", "result": result}, + "in_progress": false, + }), + stdout, + )?; + } else { + writeln!(stdout, "Operation-ID: {id}")?; + match &result.completion { + MaintenanceCompletion::Success { .. } => writeln!( + stdout, + "Original optimization completed; exact result retained." + )?, + MaintenanceCompletion::Refused { code, message } => { + writeln!(stdout, "Original optimization refused: {code}: {message}")? + } + MaintenanceCompletion::Interrupted { + publication: MaintenancePublication::Unpublished, + } => writeln!( + stdout, + "Original optimization interrupted before graph publication; receipt reconciled." + )?, + MaintenanceCompletion::Interrupted { + publication: MaintenancePublication::Published { report }, + } => writeln!( + stdout, + "Original optimization published commit {}; later derived work is not proven complete. Receipt reconciled.", + report.after_commit + )?, + } + } + stdout.flush()?; + Ok(()) + })(); + delivery.wrap_err(custody.clone())?; + // Release only after native reconciliation and exact-result delivery. The + // admission closes and settles its I/O scope; uncertainty retains the lock. + tokio::time::timeout( + std::time::Duration::from_secs(45), + admission.release_after_settlement(), + ) + .await + .wrap_err(custody.clone())? + .map_err(diagnostic) + .wrap_err(custody)?; + Ok(Completion::success()) +} + pub(crate) async fn run_full_text_rebuild( args: RebuildFullTextIndexesArgs, scope: MaintenanceScope<'_>, diff --git a/crates/omnigraph-cli/src/commands/maintenance/remote.rs b/crates/omnigraph-cli/src/commands/maintenance/remote.rs new file mode 100644 index 000000000..836154374 --- /dev/null +++ b/crates/omnigraph-cli/src/commands/maintenance/remote.rs @@ -0,0 +1,469 @@ +//! One submission to the owning server, then original-identity observation only. + +use std::io::Write; +use std::time::Duration; + +use color_eyre::eyre::{Result, WrapErr, bail}; +use omnigraph::db::OptimizeLimits; +use omnigraph_cluster::{ + MaintenanceCompletion, MaintenanceKind, MaintenanceLookup, MaintenanceRequest, + MaintenanceStatus, +}; +use reqwest::Method; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use tokio::time::Instant; + +use super::{MaintenanceScope, OptimizeArgs}; +use crate::app::Completion; +use crate::graph_http::GraphHttpClient; +use crate::helpers::{apply_bearer_token, remote_response_json_bounded, remote_url}; +use crate::output::{write_json, write_optimize}; +use crate::targets::TargetBinding; + +const RESPONSE_LIMIT: usize = 2 * 1024 * 1024; +const REQUEST_DEADLINE: Duration = Duration::from_secs(10); +const POLL_INTERVAL: Duration = Duration::from_millis(500); + +#[derive(Debug)] +pub(crate) struct WaitTimeout; + +impl std::fmt::Display for WaitTimeout { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("optimization observation deadline reached") + } +} +impl std::error::Error for WaitTimeout {} + +#[derive(Deserialize)] +struct StatusResponse { + status: MaintenanceStatus, + limits: OptimizeLimits, +} + +#[derive(Debug, Deserialize, Serialize)] +struct OperationResponse { + maintenance: MaintenanceLookup, + in_progress: bool, +} + +impl OperationResponse { + fn verify(&self, id: &str, graph: &str, expected: Option<&MaintenanceRequest>) -> Result<()> { + let (actual_id, actual_graph, request) = match &self.maintenance { + MaintenanceLookup::Outstanding { + operation_id, + graph, + invocation, + } => (operation_id, Some(graph), Some(&invocation.request)), + MaintenanceLookup::Complete { result } => ( + &result.operation_id, + Some(&result.graph), + Some(&result.invocation.request), + ), + MaintenanceLookup::ResultExpired { operation_id } + | MaintenanceLookup::NotRecorded { operation_id } + | MaintenanceLookup::IdentityMismatch { operation_id } + | MaintenanceLookup::DifferentLedger { operation_id } => (operation_id, None, None), + }; + if actual_id != id + || actual_graph.is_some_and(|actual| actual != graph) + || request.is_some_and(|request| { + request.operation_id != id + || request.graph != graph + || expected.is_some_and(|expected| expected != request) + }) + { + bail!( + "server returned a different optimization identity or request; retain Operation-ID: {id}" + ); + } + Ok(()) + } + + fn pending(&self) -> bool { + self.in_progress + && matches!( + self.maintenance, + MaintenanceLookup::Outstanding { .. } | MaintenanceLookup::NotRecorded { .. } + ) + } + + fn accepted(&self) -> bool { + matches!( + self.maintenance, + MaintenanceLookup::Outstanding { .. } | MaintenanceLookup::Complete { .. } + ) + } + + fn finish( + &self, + id: &str, + graph: &str, + json: bool, + stdout: &mut dyn Write, + stderr: &mut dyn Write, + ) -> Result { + if json { + write_json(self, stdout)?; + } else { + writeln!(stdout, "Operation-ID: {id}")?; + match &self.maintenance { + MaintenanceLookup::Complete { result } => match &result.completion { + MaintenanceCompletion::Success { report } => { + write_optimize(graph, &report.tables, false, stdout)?; + writeln!( + stdout, + "Graph commit: {} → {}", + report.before_commit, report.after_commit + )?; + } + MaintenanceCompletion::Refused { code, message } => { + writeln!(stdout, "Optimization refused: {code}: {message}")? + } + MaintenanceCompletion::Interrupted { publication } => match publication { + omnigraph_cluster::MaintenancePublication::Unpublished => writeln!( + stdout, + "Optimization interrupted before graph publication; original receipt reconciled." + )?, + omnigraph_cluster::MaintenancePublication::Published { report } => { + writeln!( + stdout, + "Optimization published commit {}; later derived work is not proven complete.", + report.after_commit + )? + } + }, + }, + MaintenanceLookup::Outstanding { .. } if self.in_progress => writeln!( + stdout, + "Optimization accepted; work continues on the server." + )?, + _ => writeln!( + stdout, + "Optimization is not proven complete; retain this operation identity." + )?, + } + } + let success = self.pending() + || matches!(&self.maintenance, + MaintenanceLookup::Complete { result } + if matches!(result.completion, MaintenanceCompletion::Success { .. })); + if !success { + writeln!( + stderr, + "Retain Operation-ID: {id}; inspect the exact result before taking further action. Do not submit a replacement for an uncertain operation." + )?; + } + stdout.flush()?; + stderr.flush()?; + Ok(if success { + Completion::success() + } else { + Completion::exit(1) + }) + } +} + +struct RemoteMaintenance { + root: String, + token: Option, + client: GraphHttpClient, + graph: String, +} + +impl RemoteMaintenance { + fn new(scope: MaintenanceScope<'_>) -> Result { + let TargetBinding::Server { endpoint, auth, .. } = &scope.target.binding else { + bail!("served optimize requires a server target"); + }; + let graph = crate::scope::graph_selection(scope.target, scope.graph)? + .ok_or_else(|| color_eyre::eyre::eyre!("served optimize requires a graph"))?; + let client = GraphHttpClient::server_target(scope.operator, scope.target)?; + let token = crate::targets::resolve_server_token(scope.operator, auth, endpoint)?; + Ok(Self { + root: endpoint.clone(), + token, + client, + graph: graph.to_owned(), + }) + } + + async fn request( + &self, + method: Method, + id: Option<&str>, + body: Option, + deadline: Instant, + ) -> Result { + let mut segments = vec!["cluster", "maintenance"]; + if let Some(id) = id { + segments.push(id); + } + let query = if method == Method::GET { + vec![("graph", self.graph.as_str())] + } else { + vec![] + }; + let url = remote_url(&self.root, &segments, &query)?; + let request = apply_bearer_token(self.client.request(method, url)?, self.token.as_deref()) + .timeout( + deadline + .saturating_duration_since(Instant::now()) + .min(REQUEST_DEADLINE), + ); + let request = match body { + Some(body) => request.json(&body), + None => request, + }; + tokio::time::timeout_at(deadline, async { + remote_response_json_bounded( + self.client.send(request).await?, + self.token.as_deref(), + Some(RESPONSE_LIMIT), + ) + .await + }) + .await? + } + + async fn observe(&self, id: &str, deadline: Instant) -> Result { + self.request(Method::GET, Some(id), None, deadline).await + } + + #[allow(clippy::too_many_arguments)] + async fn wait( + &self, + id: &str, + mut last: Option, + expected: Option<&MaintenanceRequest>, + args: &OptimizeArgs, + deadline: Instant, + stdout: &mut dyn Write, + stderr: &mut dyn Write, + ) -> Result { + loop { + if let Some(response) = &last { + response.verify(id, &self.graph, expected)?; + if (args.no_wait && response.accepted()) || !response.pending() { + return response.finish(id, &self.graph, args.json, stdout, stderr); + } + } + if Instant::now() >= deadline { + return wait_timeout(Some(id), last.as_ref(), args.json, stdout, stderr); + } + if last.is_some() { + tokio::time::sleep_until((Instant::now() + POLL_INTERVAL).min(deadline)).await; + } + if Instant::now() >= deadline { + return wait_timeout(Some(id), last.as_ref(), args.json, stdout, stderr); + } + match self.observe(id, deadline).await { + Ok(response) => last = Some(response), + Err(error) if crate::cluster_remote::retry_observation(&error) => { + tokio::time::sleep_until((Instant::now() + POLL_INTERVAL).min(deadline)).await; + } + Err(error) => return Err(error.wrap_err(recovery(id))), + } + } + } +} + +fn recovery(id: &str) -> String { + format!( + "Operation-ID: {id}; observe with optimize --target --graph --operation-id {id}; do not resubmit uncertain work" + ) +} + +fn wait_timeout( + id: Option<&str>, + last: Option<&OperationResponse>, + json: bool, + stdout: &mut dyn Write, + stderr: &mut dyn Write, +) -> Result { + (|| -> Result<()> { + if json { + write_json( + &json!({"operation_id":id,"outcome":"wait_timeout","last_observation":last}), + stdout, + )?; + } + match id { + Some(id) => writeln!( + stderr, + "Observation deadline reached; server work was not cancelled. {}", + recovery(id) + )?, + None => writeln!( + stderr, + "Observation deadline reached before submission; no optimization was sent." + )?, + } + stdout.flush()?; + stderr.flush()?; + Ok(()) + })() + .wrap_err_with(|| { + id.map(recovery) + .unwrap_or_else(|| "no optimization was sent".into()) + })?; + Err(WaitTimeout.into()) +} + +pub(crate) async fn run( + args: OptimizeArgs, + scope: MaintenanceScope<'_>, + stdout: &mut dyn Write, + stderr: &mut dyn Write, +) -> Result { + if let Some(id) = args.operation_id.as_deref() + && (id.is_empty() || id.len() > 75 || id.chars().any(char::is_control)) + { + bail!("invalid operation identity; no optimization was sent"); + } + let remote = RemoteMaintenance::new(scope)?; + let deadline = Instant::now() + Duration::from_secs(args.timeout.unwrap_or(300)); + if let Some(id) = args.operation_id.as_deref() { + // This form is observational even if the ID has never been recorded. + return remote + .wait(id, None, None, &args, deadline, stdout, stderr) + .await; + } + let status: StatusResponse = match remote.request(Method::GET, None, None, deadline).await { + Ok(status) => status, + Err(error) if error.is::() || Instant::now() >= deadline => { + return wait_timeout(None, None, args.json, stdout, stderr); + } + Err(error) => return Err(error), + }; + if let Some(id) = &status.status.outstanding_id { + bail!("an optimization is already outstanding; {}", recovery(id)); + } + status.limits.validate()?; + let request = MaintenanceRequest { + operation_id: status.status.next_operation_id(), + graph: remote.graph.clone(), + kind: MaintenanceKind::Optimize, + limits: status.limits, + }; + let id = &request.operation_id; + // Flush before the only possible POST; output failure cannot hide a new ID. + writeln!(stderr, "Operation-ID: {id}") + .and_then(|()| stderr.flush()) + .wrap_err_with(|| { + format!("could not deliver Operation-ID: {id}; no optimization was sent") + })?; + let response = remote + .request( + Method::POST, + None, + Some(serde_json::to_value(&request)?), + deadline, + ) + .await; + let last = match response { + Ok(response) => Some(response), + Err(error) if crate::cluster_remote::retry_observation(&error) => None, + Err(error) => return Err(error.wrap_err(recovery(id))), + }; + // A missing response only permits GETs for this same ID. Even no-wait + // needs evidence of acceptance before it can report success. + remote + .wait(id, last, Some(&request), &args, deadline, stdout, stderr) + .await + .wrap_err_with(|| recovery(id)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::managed_http_fixture::{IntentApiFixture, IntentReply}; + use crate::targets::{ResolvedTarget, ServerAuth, TargetProvenance}; + + struct FailedFlush; + + impl Write for FailedFlush { + fn write(&mut self, value: &[u8]) -> std::io::Result { + Ok(value.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Err(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "closed output", + )) + } + } + + #[tokio::test] + async fn identity_delivery_failure_prevents_the_only_submission() { + let root = tempfile::tempdir().unwrap(); + let input = crate::operator::InvocationConfig::at(Some(root.path().to_owned()), None); + let api = IntentApiFixture::graph(vec![IntentReply::json( + 200, + json!({ + "status":{"ledger_id":"01ARZ3NDEKTSV4RRFFQ69G5FAV","next_sequence":1,"outstanding_id":null}, + "limits":{"max_tables":1,"max_fragments":10,"max_rows":100,"max_source_bytes":10000,"max_files":20} + }), + )]); + crate::operator_routing::test_transport_scope(&[&api.origin], async { + let target = ResolvedTarget::new( + TargetBinding::Server { + endpoint: api.origin.clone(), + auth: ServerAuth::Anonymous, + default_graph: Some("knowledge".into()), + }, + TargetProvenance::Explicit { + name: "fixture".into(), + }, + ); + let scope = MaintenanceScope { + operator: &input, + target: &target, + graph: None, + actor: None, + yes: false, + }; + let error = run( + OptimizeArgs { + no_wait: true, + operation_id: None, + writers_stopped: false, + timeout: Some(5), + json: true, + }, + scope, + &mut Vec::new(), + &mut FailedFlush, + ) + .await + .unwrap_err(); + assert!( + error.to_string().contains("no optimization was sent"), + "{error:?}" + ); + let requests = api.workflow_requests(); + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].method, "GET"); + api.assert_complete(); + }) + .await; + } + + #[test] + fn timeout_exit_requires_delivery_of_the_recovery_identity() { + let id = "01ARZ3NDEKTSV4RRFFQ69G5FAV:1:01ARZ3NDEKTSV4RRFFQ69G5FAW"; + let error = + wait_timeout(Some(id), None, true, &mut Vec::new(), &mut FailedFlush).unwrap_err(); + assert!(!error.is::()); + assert!(error.to_string().contains(id)); + let mut out = Vec::new(); + let mut err = Vec::new(); + let error = wait_timeout(Some(id), None, true, &mut out, &mut err).unwrap_err(); + assert!(error.is::()); + assert_eq!( + serde_json::from_slice::(&out).unwrap()["operation_id"], + id + ); + assert!(String::from_utf8(err).unwrap().contains("not cancelled")); + } +} diff --git a/crates/omnigraph-cli/src/planes.rs b/crates/omnigraph-cli/src/planes.rs index 991e2bf92..1a4771a1e 100644 --- a/crates/omnigraph-cli/src/planes.rs +++ b/crates/omnigraph-cli/src/planes.rs @@ -45,7 +45,7 @@ pub(crate) fn requirement(command: &Command) -> Requirement { catalogue: true, }, - Command::Mutate(_) | Command::Load(_) => Graph { + Command::Mutate(_) | Command::Load(_) | Command::Optimize(_) => Graph { direct_only: false, root: true, @@ -92,7 +92,6 @@ pub(crate) fn requirement(command: &Command) -> Requirement { Command::Schema { command: SchemaCommand::Plan { .. }, } - | Command::Optimize(_) | Command::RebuildFullTextIndexes(_) | Command::Repair(_) | Command::Cleanup(_) => Graph { @@ -274,6 +273,7 @@ fn accepts_actor(command: &Command) -> bool { // The bridge asserts the actor on every request it relays. Command::Mutate(_) | Command::Load(_) + | Command::Optimize(_) | Command::RebuildFullTextIndexes(_) | Command::Mcp(_) => true, // The actor a direct target's policy decides for. @@ -337,7 +337,6 @@ fn accepts_actor(command: &Command) -> bool { | Command::Organization { .. } | Command::Init(_) | Command::Upgrade(_) - | Command::Optimize(_) | Command::Repair(_) | Command::Cleanup(_) | Command::Lint(_) @@ -363,6 +362,35 @@ pub(crate) fn validate_target( use Requirement::*; use TargetBinding::*; let need = requirement(&cli.command); + if let Command::Optimize(args) = &cli.command { + match &target.binding { + Managed(_) => { + return Err(crate::targets::refusal( + "target_kind_unsupported", + "optimize is not available for a managed target until maintenance capacity admission is enabled", + )); + } + Server { .. } if args.writers_stopped => { + bail!("optimization recovery requires an explicit direct-root --target"); + } + Server { .. } if cli.as_actor.is_some() => { + bail!("served optimize derives its actor from the credential; remove --as"); + } + Server { .. } => {} + DirectRoot { .. } if args.writers_stopped => { + if !target.is_explicit() { + bail!("optimization recovery requires an explicit direct-root --target"); + } + } + _ if args.writers_stopped => { + bail!("optimization recovery requires an explicit direct-root --target"); + } + _ if args.no_wait || args.operation_id.is_some() || args.timeout.is_some() => { + bail!("--no-wait, --operation-id and --timeout require a server target"); + } + _ => {} + } + } if matches!(need, ServedGraph | ServedTarget) && !matches!(target.binding, Server { .. }) { let label = command_label(&cli.command); let why = match &target.binding { @@ -774,7 +802,7 @@ mod tests { (&["diff", "--from", "a", "--to", "b"], false), (&["changes", "poll"], false), (&["changes", "baseline", "--out", "out"], false), - (&["optimize"], false), + (&["optimize"], true), (&["cleanup"], false), (&["mcp"], true), (&["mcp", "--write"], true), diff --git a/crates/omnigraph-cli/tests/integration/cli_cluster.rs b/crates/omnigraph-cli/tests/integration/cli_cluster.rs index 719a4e415..dc6b944ec 100644 --- a/crates/omnigraph-cli/tests/integration/cli_cluster.rs +++ b/crates/omnigraph-cli/tests/integration/cli_cluster.rs @@ -3005,6 +3005,402 @@ fn optimize_resolves_a_cluster_graph_by_id() { ); } +fn maintenance_status() -> Value { + serde_json::json!({"status":{"ledger_id":"01ARZ3NDEKTSV4RRFFQ69G5FAV","next_sequence":1,"outstanding_id":null}, + "limits":{"max_tables":32,"max_fragments":1000,"max_rows":100000,"max_source_bytes":10000000,"max_files":2000}}) +} + +fn maintenance_receipt(request: &Value, complete: bool) -> Value { + let invocation = serde_json::json!({"request":request,"actor":"act-writer","canonical_root":"file:///server-only-storage", + "result_revision":1,"input":null,"publication":null,"settlement":null,"schema_contract":{"source_hash":"schema","schema_ir_hash":"ir","schema_identity_domain":"domain","schema_identity_version":1}}); + if complete { + serde_json::json!({"maintenance":{"status":"complete","result":{ + "operation_id":request["operation_id"],"graph":request["graph"],"invocation":invocation, + "outcome":"success","report":{"before_commit":"before","after_commit":"after","tables":[], + "source_tables":0,"source_fragments":0,"source_rows":0,"source_bytes":0,"source_files":0}}},"in_progress":false}) + } else { + serde_json::json!({"maintenance":{"status":"outstanding","operation_id":request["operation_id"], + "graph":request["graph"],"invocation":invocation},"in_progress":true}) + } +} + +#[test] +fn served_optimize_submits_once_and_observes_the_original_identity() { + for mode in [ + "wait", + "no_wait", + "lost_response", + "mismatched_request", + "no_wait_before_acceptance", + ] { + let mut submitted = None; + let mut lookups = 0; + let api = IntentApiFixture::graph_responding( + match mode { + "no_wait" => 2, + "no_wait_before_acceptance" => 4, + _ => 3, + }, + move |request| match (request.method.as_str(), request.path.as_str()) { + ("GET", "/cluster/maintenance?graph=knowledge") => { + IntentReply::json(200, maintenance_status()) + } + ("POST", "/cluster/maintenance") => { + assert!(submitted.is_none(), "submission must never replay"); + submitted = Some(request.body.clone()); + if matches!( + mode, + "lost_response" | "mismatched_request" | "no_wait_before_acceptance" + ) { + IntentReply::json(504, serde_json::json!({"error":"lost response"})) + } else { + IntentReply::json(202, maintenance_receipt(&request.body, false)) + } + } + ("GET", path) => { + lookups += 1; + let body = submitted.as_ref().unwrap(); + assert_eq!( + path, + format!( + "/cluster/maintenance/{}?graph=knowledge", + body["operation_id"].as_str().unwrap() + ) + ); + if mode == "no_wait_before_acceptance" && lookups == 1 { + return IntentReply::json( + 200, + serde_json::json!({"maintenance":{"status":"not_recorded","operation_id":body["operation_id"]},"in_progress":true}), + ); + } + let mut receipt = + maintenance_receipt(body, mode != "no_wait_before_acceptance"); + if mode == "mismatched_request" { + receipt["maintenance"]["result"]["invocation"]["request"]["limits"]["max_rows"] = + 1.into(); + } + IntentReply::json(200, receipt) + } + _ => panic!("unexpected request: {request:?}"), + }, + ); + let operator = KeyedServer::new(&api.origin); + let mut command = cli(); + command.envs(operator.env("optimization-token")).args([ + "optimize", + "--target", + KEYED_SERVER_NAME, + "--graph", + "knowledge", + "--json", + "--timeout", + "5", + ]); + if mode.starts_with("no_wait") { + command.arg("--no-wait"); + } + let output = command.output().unwrap(); + assert_eq!( + output.status.success(), + mode != "mismatched_request", + "{mode}: {output:?}" + ); + let requests = api.workflow_requests(); + let posts: Vec<_> = requests + .iter() + .filter(|request| request.method == "POST") + .collect(); + assert_eq!(posts.len(), 1); + let id = posts[0].body["operation_id"].as_str().unwrap(); + assert!(id.starts_with("01ARZ3NDEKTSV4RRFFQ69G5FAV:1:")); + assert_eq!(id.len(), 55); + assert!(String::from_utf8_lossy(&output.stderr).contains(id)); + assert_eq!(posts[0].body["limits"], maintenance_status()["limits"]); + assert_eq!(posts[0].body["kind"], "optimize"); + for request in &requests { + assert_eq!( + request.headers.get("authorization").map(String::as_str), + Some("Bearer optimization-token") + ); + } + if mode != "mismatched_request" { + assert_eq!( + parse_stdout_json(&output)["maintenance"]["status"], + if mode.starts_with("no_wait") { + "outstanding" + } else { + "complete" + } + ); + } + assert_no_core_effects(operator.home()); + api.assert_complete(); + } +} + +#[test] +fn served_optimize_operation_id_is_get_only_even_when_not_recorded_or_expired() { + let id = "01ARZ3NDEKTSV4RRFFQ69G5FAV:1:01ARZ3NDEKTSV4RRFFQ69G5FAW"; + for status in [ + "not_recorded", + "result_expired", + "identity_mismatch", + "different_ledger", + ] { + let response = serde_json::json!({"maintenance":{"status":status,"operation_id":id},"in_progress":false}); + let api = IntentApiFixture::graph(vec![IntentReply::json(200, response.clone())]); + let output = output_failure( + cli() + .args(["optimize", "--target"]) + .fixture_target(&api.origin) + .args(["--graph", "knowledge", "--operation-id", id, "--json"]), + ); + assert_eq!(parse_stdout_json(&output), response); + let requests = api.workflow_requests(); + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].method, "GET"); + assert_eq!( + requests[0].path, + format!("/cluster/maintenance/{id}?graph=knowledge") + ); + api.assert_complete(); + } +} + +#[test] +fn served_optimize_timeout_never_cancels_or_resubmits() { + let id = "01ARZ3NDEKTSV4RRFFQ69G5FAV:1:01ARZ3NDEKTSV4RRFFQ69G5FAW"; + let api = IntentApiFixture::graph_with_response_delay( + vec![IntentReply::json( + 200, + serde_json::json!({"maintenance":{"status":"not_recorded","operation_id":id},"in_progress":false}), + )], + std::time::Duration::from_secs(2), + ); + let output = output_failure( + cli() + .args(["optimize", "--target"]) + .fixture_target(&api.origin) + .args([ + "--graph", + "knowledge", + "--operation-id", + id, + "--timeout", + "1", + "--json", + ]), + ); + assert_eq!(output.status.code(), Some(5), "{output:?}"); + assert_eq!(parse_stdout_json(&output)["outcome"], "wait_timeout"); + let requests = api.workflow_requests(); + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].method, "GET"); + api.assert_complete(); +} + +#[test] +fn served_optimize_preserves_refusal_and_unresolved_receipts_without_success() { + let id = "01ARZ3NDEKTSV4RRFFQ69G5FAV:1:01ARZ3NDEKTSV4RRFFQ69G5FAW"; + let request = serde_json::json!({"operation_id":id,"graph":"knowledge","kind":"optimize","limits":maintenance_status()["limits"]}); + for outcome in ["refused", "outstanding", "unpublished", "published"] { + let mut response = maintenance_receipt(&request, outcome != "outstanding"); + response["in_progress"] = false.into(); + if outcome == "refused" { + let result = response["maintenance"]["result"].as_object_mut().unwrap(); + result.remove("report"); + result.insert("outcome".into(), "refused".into()); + result.insert("code".into(), "optimization_preflight_refused".into()); + result.insert("message".into(), "source exceeds configured ceiling".into()); + } else if matches!(outcome, "unpublished" | "published") { + let result = response["maintenance"]["result"].as_object_mut().unwrap(); + let report = result.remove("report").unwrap(); + result.insert("outcome".into(), "interrupted".into()); + result.insert( + "publication".into(), + if outcome == "published" { + serde_json::json!({"status":"published","report":report}) + } else { + serde_json::json!({"status":"unpublished"}) + }, + ); + } + let api = IntentApiFixture::graph(vec![IntentReply::json(200, response.clone())]); + let output = output_failure( + cli() + .args(["optimize", "--target"]) + .fixture_target(&api.origin) + .args([ + "--graph", + "knowledge", + "--operation-id", + id, + "--no-wait", + "--json", + ]), + ); + assert_eq!(parse_stdout_json(&output), response); + assert_eq!(api.workflow_requests().len(), 1); + assert_eq!(api.workflow_requests()[0].method, "GET"); + api.assert_complete(); + } +} + +#[test] +fn optimize_refuses_inapplicable_flags_and_managed_targets_before_requests() { + let api = IntentApiFixture::graph(vec![]); + for extra in [ + vec!["--as", "act-other"], + vec!["--branch", "review"], + vec!["--timeout", "0"], + vec!["--timeout", "3601"], + vec!["--operation-id", "original", "--writers-stopped"], + ] { + output_failure( + cli() + .args(["optimize", "--target"]) + .fixture_target(&api.origin) + .args(["--graph", "knowledge"]) + .args(extra), + ); + } + let root = tempdir().unwrap(); + for args in [ + vec!["--no-wait"], + vec!["--operation-id", "original"], + vec!["--timeout", "1"], + vec!["--writers-stopped"], + vec![ + "--operation-id", + "original", + "--writers-stopped", + "--no-wait", + ], + vec![ + "--operation-id", + "original", + "--writers-stopped", + "--timeout", + "1", + ], + ] { + output_failure( + cli() + .args(["optimize", "--target"]) + .arg(root.path().join("must-not-create")) + .args(args), + ); + assert!(!root.path().join("must-not-create").exists()); + } + let managed = IntentApiFixture::new(vec![]); + let output = output_failure( + managed_fixture_command(root.path(), &managed.origin, "managed-test") + .arg("optimize") + .args(["--graph", "knowledge", "--json"]), + ); + assert!( + String::from_utf8_lossy(&output.stdout).contains("target_kind_unsupported"), + "{output:?}" + ); + assert!(managed.workflow_requests().is_empty()); + api.assert_complete(); + managed.assert_complete(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn optimize_recovery_requires_exclusion_and_reconciles_the_original_receipt() { + use omnigraph_cluster::{ + ClusterAdmissionPurpose, IdentityAuthorization, MaintenanceAcceptance, MaintenanceKind, + MaintenanceRequest, + }; + let temp = tempdir().unwrap(); + write_cluster_config_fixture(temp.path()); + let policy = temp.path().join("base.policy.yaml"); + fs::write( + &policy, + fs::read_to_string(&policy).unwrap().replace( + "actions: [change, schema_apply]", + "actions: [change, schema_apply, optimize]", + ), + ) + .unwrap(); + apply_cluster_fixture(temp.path()); + let root = temp.path().to_str().unwrap(); + let actor = IdentityAuthorization::authenticated("act-cluster-test").unwrap(); + let owner = omnigraph_cluster::acquire_cluster_admission(root, ClusterAdmissionPurpose::Serve) + .await + .unwrap() + .unwrap(); + let id = omnigraph_cluster::maintenance_status(&owner, "knowledge", &actor) + .await + .unwrap() + .next_operation_id(); + let request = MaintenanceRequest { + operation_id: id.clone(), + graph: "knowledge".into(), + kind: MaintenanceKind::Optimize, + limits: serde_json::from_value(maintenance_status()["limits"].clone()).unwrap(), + }; + let MaintenanceAcceptance::Accepted(token) = + omnigraph_cluster::accept_maintenance(&owner, request, &actor) + .await + .unwrap() + else { + panic!("fresh acceptance") + }; + let command = || { + let mut command = cli(); + command.args([ + "optimize", + "--target", + root, + "--graph", + "knowledge", + "--as", + "act-cluster-test", + "--operation-id", + &id, + "--writers-stopped", + "--json", + ]); + command + }; + let refused = output_failure(&mut command()); + assert!( + String::from_utf8_lossy(&refused.stdout).contains("state_lock_held") + || String::from_utf8_lossy(&refused.stderr).contains("state_lock_held"), + "{refused:?}" + ); + // This fixture accepted no physical work. Drop every owner before the + // explicit administrator unlock; the recovery flag never unlocks it. + let lock = owner.lock_id().to_string(); + drop(token); + drop(owner); + omnigraph_cluster::force_unlock_storage_root(root, &lock) + .await + .unwrap(); + let journal = temp.path().join("__cluster/maintenance.json"); + let before = fs::read(&journal).unwrap(); + let recovered = parse_stdout_json(&output_success(&mut command())); + assert_eq!(recovered["maintenance"]["result"]["operation_id"], id); + assert_eq!(recovered["maintenance"]["result"]["outcome"], "interrupted"); + assert_eq!( + recovered["maintenance"]["result"]["publication"]["status"], + "unpublished" + ); + assert_ne!(fs::read(&journal).unwrap(), before); + assert_eq!( + parse_stdout_json(&output_success(&mut command())), + recovered + ); + let successor = + omnigraph_cluster::acquire_cluster_admission(root, ClusterAdmissionPurpose::Serve) + .await + .unwrap() + .unwrap(); + successor.release_after_settlement().await.unwrap(); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn v2_root_admission_blocks_cli_write_and_native_control_doors() { let temp = applied_knowledge_cluster(); diff --git a/crates/omnigraph-cli/tests/integration/cli_data.rs b/crates/omnigraph-cli/tests/integration/cli_data.rs index 939fe3de9..18a81e1fe 100644 --- a/crates/omnigraph-cli/tests/integration/cli_data.rs +++ b/crates/omnigraph-cli/tests/integration/cli_data.rs @@ -1005,7 +1005,9 @@ fn optimize_json_succeeds_on_local_graph() { assert!(pending.iter().all(|index| index.get("column").is_none())); let human = stdout_string(&output_success( - cli().arg("optimize").arg("--target").arg(&graph), + cli() + .args(["optimize", "--as", "act-cli-optimize", "--target"]) + .arg(&graph), )); assert!(human.contains(" datasets"), "{human}"); assert!(human.contains("node type 'Person'"), "{human}"); @@ -1096,15 +1098,9 @@ fn optimize_json_succeeds_on_local_graph() { } #[test] -fn maintenance_refuses_server_targets_before_network() { +fn direct_only_maintenance_refuses_server_targets_before_network() { let server = crate::support::managed_http::IntentApiFixture::graph(Vec::new()); - for command in [ - "optimize", - "rebuild-full-text-indexes", - "repair", - "cleanup", - "upgrade", - ] { + for command in ["rebuild-full-text-indexes", "repair", "cleanup", "upgrade"] { let output = output_failure( cli() .arg(command) @@ -1122,12 +1118,6 @@ fn maintenance_refuses_server_targets_before_network() { server.assert_complete(); } -#[test] -fn optimize_with_as_flag_errors() { - let output = output_failure(cli().args(["optimize", "--as", "act-op"])); - assert!(String::from_utf8_lossy(&output.stderr).contains("optimize does not accept --as")); -} - #[test] fn retired_address_flags_refuse_before_resolving_a_target() { for flag in ["--store", "--server", "--cluster", "--profile", "--uri"] { diff --git a/crates/omnigraph-cli/tests/integration/system_remote.rs b/crates/omnigraph-cli/tests/integration/system_remote.rs index dc9e483a5..2f97002bf 100644 --- a/crates/omnigraph-cli/tests/integration/system_remote.rs +++ b/crates/omnigraph-cli/tests/integration/system_remote.rs @@ -1155,6 +1155,129 @@ fn remote_blob_write_delivery_loss_never_replays_committed_effect() { } } +#[test] +fn remote_optimize_preserves_data_and_observes_the_exact_receipt() { + const TOKEN: &str = "optimize-wire-token"; + let cluster = converged_loaded_cluster(GRAPH_ID, None); + fs::write( + cluster.path().join("graph.policy.yaml"), + "version: 1\ngroups:\n readers: [act-cluster-test, act-reader]\n operators: [act-cluster-test]\nprotected_branches: [main]\nrules:\n - id: read\n allow:\n actors: { group: readers }\n actions: [read, invoke_query]\n - id: optimize\n allow:\n actors: { group: operators }\n actions: [optimize]\n target_branch_scope: protected\n", + ).unwrap(); + let config_path = cluster.path().join("cluster.yaml"); + let config = fs::read_to_string(&config_path).unwrap(); + fs::write(&config_path, format!("{config}policies:\n graph:\n file: ./graph.policy.yaml\n applies_to: [{GRAPH_ID}]\n")).unwrap(); + apply_cluster_fixture(cluster.path()); + let limits_path = cluster.path().join("maintenance.yaml"); + fs::write(&limits_path, "max_tables: 16\nmax_fragments: 100\nmax_rows: 1000\nmax_source_bytes: 10485760\nmax_files: 1000\n").unwrap(); + let server = spawn_server_with_cluster_env( + cluster.path(), + &[ + ( + "OMNIGRAPH_SERVER_BEARER_TOKENS_JSON", + r#"{"act-cluster-test":"optimize-wire-token","act-reader":"reader-token"}"#, + ), + ( + "OMNIGRAPH_MAINTENANCE_CONFIG", + limits_path.to_str().unwrap(), + ), + ], + ); + let client = graph_http_client(); + let status = || { + client + .get(format!("{}/cluster/maintenance", server.base_url)) + .query(&[("graph", GRAPH_ID)]) + .bearer_auth(TOKEN) + .send() + .unwrap() + .error_for_status() + .unwrap() + .json::() + .unwrap() + }; + let query_path = cluster.path().join("all-people.gq"); + fs::write(&query_path, "query all_people() {\n match { $p: Person }\n return { $p.name, $p.age }\n order { $p.name asc }\n}\n").unwrap(); + let read = || { + parse_stdout_json(&output_success( + cli() + .envs(server.credential(TOKEN)) + .args(["--target"]) + .fixture_target(&server.base_url) + .args(["--graph", GRAPH_ID, "query", "--query"]) + .arg(&query_path) + .args(["all_people", "--json"]), + )) + }; + let before = read(); + assert_eq!(before["row_count"], 4); + let before_status = status(); + let denied = output_failure( + cli() + .envs(server.credential("reader-token")) + .arg("--target") + .fixture_target(&server.base_url) + .args(["--graph", GRAPH_ID, "optimize", "--json"]), + ); + let denial = parse_stdout_json(&denied); + assert_eq!(denial["code"], "forbidden", "{denial}"); + assert!(denial["error"].as_str().unwrap().contains("optimize")); + assert!(!String::from_utf8_lossy(&denied.stderr).contains("Operation-ID:")); + assert_eq!( + status(), + before_status, + "denial must not reserve an identity" + ); + assert_eq!(read(), before, "denial must not change graph data or head"); + + let output = output_success( + cli() + .envs(server.credential(TOKEN)) + .arg("--target") + .fixture_target(&server.base_url) + .args(["--graph", GRAPH_ID, "optimize", "--json"]), + ); + let receipt = parse_stdout_json(&output); + assert_eq!(receipt["maintenance"]["status"], "complete"); + let result = &receipt["maintenance"]["result"]; + assert_eq!(result["outcome"], "success"); + assert_eq!(result["graph"], GRAPH_ID); + assert_eq!(result["invocation"]["actor"], "act-cluster-test"); + let id = result["operation_id"].as_str().unwrap(); + assert!(String::from_utf8_lossy(&output.stderr).contains(&format!("Operation-ID: {id}"))); + let report = &result["report"]; + let after = read(); + assert_eq!(report["before_commit"], before["graph_commit_id"]); + assert_eq!(report["after_commit"], after["graph_commit_id"]); + assert_eq!(after["rows"], before["rows"]); + assert_eq!(after["row_count"], before["row_count"]); + let after_status = status(); + assert_eq!( + after_status["status"]["next_sequence"].as_u64().unwrap(), + before_status["status"]["next_sequence"].as_u64().unwrap() + 1 + ); + let observed = parse_stdout_json(&output_success( + cli() + .envs(server.credential(TOKEN)) + .arg("--target") + .fixture_target(&server.base_url) + .args([ + "--graph", + GRAPH_ID, + "optimize", + "--operation-id", + id, + "--json", + ]), + )); + assert_eq!(observed["maintenance"], receipt["maintenance"]); + assert_eq!( + status(), + after_status, + "lookup must not admit another operation" + ); + assert_eq!(read(), after, "lookup must not move the graph head"); +} + #[test] fn remote_deployment_delivery_loss_preserves_owned_completion() { use crate::support::managed_http::{DeploymentDeliveryFault, IntentApiFixture}; diff --git a/crates/omnigraph-cli/tests/support/managed_http.rs b/crates/omnigraph-cli/tests/support/managed_http.rs index 017ca25e9..074f5add0 100644 --- a/crates/omnigraph-cli/tests/support/managed_http.rs +++ b/crates/omnigraph-cli/tests/support/managed_http.rs @@ -63,6 +63,8 @@ enum Forwarding { Deployment(String, DeploymentDeliveryFault), } +type Responder = Box IntentReply + Send>; + impl Forwarding { fn upstream(&self) -> &str { match self { @@ -103,6 +105,25 @@ impl IntentApiFixture { Self::start(replies, None, Duration::ZERO, true) } + /// Echo identities allocated by the actual client while retaining exact + /// request counts and the same transport/discovery fixture. + pub fn graph_responding( + expected_requests: usize, + responder: impl FnMut(&IntentRequest) -> IntentReply + Send + 'static, + ) -> Self { + let mut fixture = Self::start_with_responder( + |_| Vec::new(), + None, + Duration::ZERO, + true, + None, + None, + Some(Box::new(responder)), + ); + fixture.reply_count = Some(expected_requests); + fixture + } + /// Change fixture state before the first discovery response without races /// between a test thread and the client receiving that response. pub fn graph_with_discovery_hook( @@ -175,12 +196,33 @@ impl IntentApiFixture { } fn start_with_origin( + replies: impl FnOnce(&str) -> Vec, + session: Option, + delay: Duration, + graph: bool, + forwarding: Option, + before_discovery: Option>, + ) -> Self { + Self::start_with_responder( + replies, + session, + delay, + graph, + forwarding, + before_discovery, + None, + ) + } + + #[allow(clippy::too_many_arguments)] + fn start_with_responder( replies: impl FnOnce(&str) -> Vec, session: Option, delay: Duration, graph: bool, forwarding: Option, mut before_discovery: Option>, + mut responder: Option, ) -> Self { use std::io::Write; use std::sync::atomic::Ordering; @@ -465,6 +507,8 @@ impl IntentApiFixture { && let Some(session) = &session_response { IntentReply::json(200, session.lock().unwrap().clone()) + } else if let Some(responder) = &mut responder { + responder(&request) } else { replies.pop_front().unwrap_or_else(|| { IntentReply::json(500, serde_json::json!({"type":"unexpected_request"})) diff --git a/crates/omnigraph-cluster/src/admission.rs b/crates/omnigraph-cluster/src/admission.rs index 259634eb2..f78dd5029 100644 --- a/crates/omnigraph-cluster/src/admission.rs +++ b/crates/omnigraph-cluster/src/admission.rs @@ -17,6 +17,7 @@ pub enum ClusterAdmissionPurpose { GraphOperation, Deployment, Reconcile { deployment_id: String }, + MaintenanceRecovery { operation_id: String }, } impl ClusterAdmissionPurpose { @@ -26,6 +27,7 @@ impl ClusterAdmissionPurpose { Self::GraphOperation => "graph_operation", Self::Deployment => "deployment", Self::Reconcile { .. } => "deployment_reconcile", + Self::MaintenanceRecovery { .. } => "maintenance_reconcile", } } } @@ -170,6 +172,21 @@ impl ClusterAdmission { Ok(()) } + pub(crate) async fn validate_maintenance_recovery( + &self, + operation_id: &str, + ) -> Result<(), Diagnostic> { + if !matches!(&self.owner().purpose, ClusterAdmissionPurpose::MaintenanceRecovery { operation_id: original } if original == operation_id) + { + return Err(Diagnostic::error( + "cluster_admission_purpose_mismatch", + "__cluster/lock.json", + "only the exact original maintenance recovery is admitted", + )); + } + self.validate_current_lock().await + } + /// The exact achieved contract captured with this serving admission. /// This lookup opens no engine. Startup compares it with each opened graph /// independently, so an unavailable graph does not block healthy peers. @@ -249,6 +266,16 @@ impl ClusterAdmission { /// Alias spellings are accepted only when they resolve inside this root's /// canonical graph layout; graph symlinks escaping that layout are refused. pub async fn validate_graph_uri(&self, graph_uri: &str) -> Result<(), Diagnostic> { + if matches!( + self.owner().purpose, + ClusterAdmissionPurpose::MaintenanceRecovery { .. } + ) { + return Err(Diagnostic::error( + "cluster_admission_purpose_mismatch", + "__cluster/lock.json", + "maintenance recovery cannot admit an ordinary graph writer", + )); + } self.admitted_graph_id(graph_uri).map(|_| ()) } @@ -348,7 +375,25 @@ pub async fn acquire_cluster_admission( purpose: ClusterAdmissionPurpose, ) -> Result, Diagnostic> { let mut store = ClusterStore::for_storage_root(storage_root)?; - if purpose == ClusterAdmissionPurpose::Serve { + if matches!( + purpose, + ClusterAdmissionPurpose::Serve | ClusterAdmissionPurpose::MaintenanceRecovery { .. } + ) { + store = store.with_io_scope(omnigraph_storage::StorageIoScope::new())?; + } + acquire_with_store(&store, purpose).await +} + +/// Capture the caller's provider binding before acquiring the exact native lock. +pub async fn acquire_cluster_admission_with_binding( + binding: &omnigraph_storage::CapturedStorageBinding, + purpose: ClusterAdmissionPurpose, +) -> Result, Diagnostic> { + let mut store = ClusterStore::for_storage_binding(binding)?; + if matches!( + purpose, + ClusterAdmissionPurpose::Serve | ClusterAdmissionPurpose::MaintenanceRecovery { .. } + ) { store = store.with_io_scope(omnigraph_storage::StorageIoScope::new())?; } acquire_with_store(&store, purpose).await @@ -385,7 +430,10 @@ pub async fn acquire_graph_admission_with_binding( .for_uri(&root) .map_err(|error| Diagnostic::error("storage_root_invalid", "storage", error.to_string()))?; let mut store = ClusterStore::for_storage_binding(&root_binding)?; - if purpose == ClusterAdmissionPurpose::Serve { + if matches!( + purpose, + ClusterAdmissionPurpose::Serve | ClusterAdmissionPurpose::MaintenanceRecovery { .. } + ) { store = store.with_io_scope(omnigraph_storage::StorageIoScope::new())?; } let admission = acquire_with_store(&store, purpose).await?; @@ -435,6 +483,15 @@ pub(crate) async fn acquire_with_store( "cluster version changed while acquiring admission", )); } + // Reacquiring the process lock does not settle an earlier optimization. + // Check under that exact lock before any successor can open graphs for + // writing. The refusal path also preserves the lock while uncertainty + // remains; neither restart nor force-unlock is maintenance recovery. + match &purpose { + ClusterAdmissionPurpose::MaintenanceRecovery { operation_id } => + crate::maintenance::require_maintenance_recovery(store, &state, operation_id).await?, + _ => crate::maintenance::require_no_maintenance(store).await?, + } let allowed = match (&purpose, state.outstanding_deployment_id()) { (ClusterAdmissionPurpose::Reconcile { deployment_id }, Some(outstanding)) => { deployment_id == outstanding diff --git a/crates/omnigraph-cluster/src/authorization.rs b/crates/omnigraph-cluster/src/authorization.rs index a992307aa..6a408e08a 100644 --- a/crates/omnigraph-cluster/src/authorization.rs +++ b/crates/omnigraph-cluster/src/authorization.rs @@ -336,7 +336,7 @@ impl AppliedPolicies { action, graph, (action == PolicyAction::Read).then_some("main"), - (action == PolicyAction::SchemaApply).then_some("main"), + matches!(action, PolicyAction::SchemaApply | PolicyAction::Optimize).then_some("main"), ) } diff --git a/crates/omnigraph-cluster/src/deployment/bootstrap_fault_tests.rs b/crates/omnigraph-cluster/src/deployment/bootstrap_fault_tests.rs index 909394217..1b0b50551 100644 --- a/crates/omnigraph-cluster/src/deployment/bootstrap_fault_tests.rs +++ b/crates/omnigraph-cluster/src/deployment/bootstrap_fault_tests.rs @@ -31,6 +31,12 @@ enum Point { StateTerminal, Claim, Release, + MaintenanceAccepted, + MaintenanceInput, + MaintenancePublication, + MaintenanceSettlement, + MaintenanceTerminal, + MaintenanceRecoveryTerminal, } #[derive(Clone, Copy, Debug)] @@ -42,6 +48,7 @@ enum Fault { #[derive(Debug)] struct Controlled { inner: ObjectStorageAdapter, + uri_prefix: Option<&'static str>, allow_fixture_setup: bool, writes: Mutex>, forbidden_calls: AtomicUsize, @@ -55,6 +62,7 @@ impl Controlled { fn new() -> Self { Self { inner: ObjectStorageAdapter::in_memory(), + uri_prefix: Some(URI_PREFIX), allow_fixture_setup: false, writes: Mutex::default(), forbidden_calls: AtomicUsize::new(0), @@ -66,8 +74,18 @@ impl Controlled { } fn key<'a>(&self, uri: &'a str) -> &'a str { - uri.strip_prefix(URI_PREFIX) - .expect("test-only S3 namespace") + match self.uri_prefix { + Some(prefix) => uri.strip_prefix(prefix).expect("test-only S3 namespace"), + None => uri, + } + } + + fn local() -> Self { + Self { + inner: ObjectStorageAdapter::local(), + uri_prefix: None, + ..Self::new() + } } fn point(uri: &str, payload: &str, cas: bool) -> Option { @@ -95,6 +113,25 @@ impl Controlled { _ => None, } } + } else if uri.ends_with("/__cluster/maintenance.json") { + let journal: serde_json::Value = serde_json::from_str(payload).unwrap(); + Some(if journal["outstanding"].is_null() { + if journal["results"].as_array().unwrap().last().unwrap()["outcome"] + == "interrupted" + { + Point::MaintenanceRecoveryTerminal + } else { + Point::MaintenanceTerminal + } + } else if !journal["outstanding"]["settlement"].is_null() { + Point::MaintenanceSettlement + } else if !journal["outstanding"]["publication"].is_null() { + Point::MaintenancePublication + } else if journal["outstanding"]["input"].is_null() { + Point::MaintenanceAccepted + } else { + Point::MaintenanceInput + }) } else { None } @@ -202,7 +239,10 @@ impl StorageAdapter for Controlled { self.inner.list_dir(self.key(uri)).await.map(|entries| { entries .into_iter() - .map(|entry| format!("{URI_PREFIX}{entry}")) + .map(|entry| match self.uri_prefix { + Some(prefix) => format!("{prefix}{entry}"), + None => entry, + }) .collect() }) } @@ -218,7 +258,10 @@ impl StorageAdapter for Controlled { .map(|entries| { entries .into_iter() - .map(|entry| format!("{URI_PREFIX}{entry}")) + .map(|entry| match self.uri_prefix { + Some(prefix) => format!("{prefix}{entry}"), + None => entry, + }) .collect() }) } @@ -321,6 +364,359 @@ fn fixture() -> ( (directory, bundle, store, adapter) } +#[tokio::test] +async fn maintenance_committed_journal_writes_do_not_replay_after_loss_or_cancellation() { + use crate::{ + IdentityAuthorization, MaintenanceAcceptance, MaintenanceCompletion, MaintenanceLookup, + accept_maintenance, bind_maintenance_input, bind_maintenance_publication, + complete_maintenance, maintenance_lookup, + }; + use omnigraph::db::OptimizeInput; + + for point in [ + Point::MaintenanceAccepted, + Point::MaintenanceInput, + Point::MaintenancePublication, + Point::MaintenanceTerminal, + ] { + for fault in [Fault::LostResponse, Fault::CancelAfterCommit] { + // Reuse the real applied graph/policy fixture and the existing + // fault adapter. Only the adapter's post-commit response is lost; + // this test does not claim cloud settlement or graph execution. + let (directory, admission) = crate::maintenance::tests::fixture().await; + admission.release_after_settlement().await.unwrap(); + let adapter = Arc::new(Controlled::local()); + let store = ClusterStore::for_storage_root(directory.path().to_str().unwrap()) + .unwrap() + .with_bootstrap_test_adapter(adapter.clone()); + let admission = + crate::admission::acquire_with_store(&store, crate::ClusterAdmissionPurpose::Serve) + .await + .unwrap() + .unwrap(); + let actor = IdentityAuthorization::authenticated("principal:owner").unwrap(); + let request = crate::maintenance::tests::request(&admission).await; + adapter.arm(point, fault); + { + let invoke = async { + let MaintenanceAcceptance::Accepted(mut accepted) = + accept_maintenance(&admission, request.clone(), &actor).await? + else { + panic!("fresh request was already recorded") + }; + bind_maintenance_input( + &admission, + &mut accepted, + OptimizeInput { + before_commit: "retained-input".into(), + report_max_bytes: 16384, + schema_contract: omnigraph::db::SchemaContractDigest { + source_hash: "source".into(), + schema_ir_hash: "ir".into(), + schema_identity_domain: "domain".into(), + schema_identity_version: 1, + }, + source_tables: 1, + source_fragments: 1, + source_rows: 1, + source_bytes: 128, + source_files: 1, + }, + ) + .await?; + if point == Point::MaintenancePublication { + bind_maintenance_publication( + &admission, + &mut accepted, + omnigraph::db::OptimizePublication { + before_manifest_version: 1, + intent_nonce: ulid::Ulid::new().to_string(), + schema_contract: omnigraph::db::SchemaContractDigest { + source_hash: "source".into(), + schema_ir_hash: "ir".into(), + schema_identity_domain: "domain".into(), + schema_identity_version: 1, + }, + actor: Some(actor.actor().into()), + created_at: 1, + report: omnigraph::db::OptimizeReport { + before_commit: "retained-input".into(), + after_commit: "retained-input".into(), + tables: Vec::new(), + source_tables: 1, + source_fragments: 1, + source_rows: 1, + source_bytes: 128, + source_files: 1, + }, + }, + ) + .await?; + } + complete_maintenance( + &admission, + accepted, + MaintenanceCompletion::Refused { + code: "fixture_preflight_refusal".into(), + message: "no graph effect was attempted".into(), + }, + ) + .await + }; + match fault { + Fault::LostResponse => { + let error = invoke.await.unwrap_err(); + assert_eq!(error.code, "maintenance_write_uncertain", "{point:?}"); + } + Fault::CancelAfterCommit => { + tokio::pin!(invoke); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + tokio::select! { + _ = adapter.committed.notified() => {} + result = &mut invoke => panic!("expected committed blocked write: {result:?}"), + } + }) + .await + .expect("real journal write must commit before cancellation"); + } + } + } + assert!(adapter.fault.lock().unwrap().is_none()); + let retained = + maintenance_lookup(&admission, "knowledge", &request.operation_id, &actor) + .await + .unwrap(); + match (&retained, point) { + (MaintenanceLookup::Complete { .. }, Point::MaintenanceTerminal) => {} + (MaintenanceLookup::Outstanding { invocation, .. }, Point::MaintenanceAccepted) => { + assert!(invocation.input.is_none()); + } + (MaintenanceLookup::Outstanding { invocation, .. }, Point::MaintenanceInput) => { + assert!(invocation.input.is_some()); + } + ( + MaintenanceLookup::Outstanding { invocation, .. }, + Point::MaintenancePublication, + ) => { + assert!(invocation.publication.is_some()); + assert!(invocation.settlement.is_none()); + } + _ => panic!("wrong retained state after {point:?}: {retained:?}"), + } + let writes = adapter.writes.lock().unwrap().len(); + let MaintenanceAcceptance::Existing(existing) = + accept_maintenance(&admission, request, &actor) + .await + .unwrap() + else { + panic!("lost response or cancellation must never mint an execution token") + }; + assert_eq!( + serde_json::to_value(existing).unwrap(), + serde_json::to_value(retained).unwrap() + ); + assert_eq!(adapter.writes.lock().unwrap().len(), writes); + drop(admission); + adapter.assert_no_forbidden_calls().await; + } + } +} + +#[tokio::test] +async fn maintenance_recovery_journal_loss_keeps_the_original_settlement_and_result() { + use crate::{ + ClusterAdmissionPurpose, IdentityAuthorization, MaintenanceAcceptance, + MaintenanceCompletion, MaintenancePublication, accept_maintenance, bind_maintenance_input, + bind_maintenance_publication, reconcile_maintenance, + }; + use omnigraph::db::{ + Omnigraph, OptimizeInput, OptimizePublication, OptimizeReport, ReadTarget, + }; + + for point in [ + Point::MaintenanceSettlement, + Point::MaintenanceRecoveryTerminal, + ] { + for fault in [Fault::LostResponse, Fault::CancelAfterCommit] { + let (directory, admission) = crate::maintenance::tests::fixture().await; + admission.release_after_settlement().await.unwrap(); + let root = directory.path().to_str().unwrap(); + let adapter = Arc::new(Controlled::local()); + let store = ClusterStore::for_storage_root(root) + .unwrap() + .with_bootstrap_test_adapter(adapter.clone()); + let admission = + crate::admission::acquire_with_store(&store, ClusterAdmissionPurpose::Serve) + .await + .unwrap() + .unwrap(); + let actor = IdentityAuthorization::authenticated("principal:owner").unwrap(); + let request = crate::maintenance::tests::request(&admission).await; + let MaintenanceAcceptance::Accepted(mut accepted) = + accept_maintenance(&admission, request.clone(), &actor) + .await + .unwrap() + else { + panic!("fresh invocation must be accepted") + }; + + // This fixture exercises the journal boundary, not optimization's + // physical work. Its prepared publication has the real base and + // contract; recovery must publish at most one neutral competitor. + let graph_uri = store.graph_root("knowledge"); + let db = Omnigraph::open(&graph_uri).await.unwrap(); + let snapshot = db.snapshot_of(ReadTarget::branch("main")).await.unwrap(); + let base_version = snapshot.graph_manifest_version(); + let before = snapshot.graph_head(None).unwrap().to_string(); + let schema_contract = db.schema_contract_digest(); + let input = OptimizeInput { + before_commit: before.clone(), + schema_contract: schema_contract.clone(), + report_max_bytes: 16384, + source_tables: 1, + source_fragments: 1, + source_rows: 1, + source_bytes: 128, + source_files: 1, + }; + bind_maintenance_input(&admission, &mut accepted, input) + .await + .unwrap(); + bind_maintenance_publication( + &admission, + &mut accepted, + OptimizePublication { + before_manifest_version: base_version, + intent_nonce: ulid::Ulid::new().to_string(), + schema_contract, + actor: Some(actor.actor().into()), + created_at: 1, + report: OptimizeReport { + before_commit: before.clone(), + after_commit: before, + tables: Vec::new(), + source_tables: 1, + source_fragments: 1, + source_rows: 1, + source_bytes: 128, + source_files: 1, + }, + }, + ) + .await + .unwrap(); + drop(snapshot); + drop(db); + drop(accepted); + let lock_id = admission.lock_id().to_string(); + drop(admission); + crate::force_unlock_storage_root(root, &lock_id) + .await + .unwrap(); + let recovery = crate::admission::acquire_with_store( + &store, + ClusterAdmissionPurpose::MaintenanceRecovery { + operation_id: request.operation_id.clone(), + }, + ) + .await + .unwrap() + .unwrap(); + adapter.arm(point, fault); + { + let invoke = + reconcile_maintenance(&recovery, "knowledge", &request.operation_id, &actor); + match fault { + Fault::LostResponse => { + let error = invoke.await.unwrap_err(); + assert_eq!(error.code, "maintenance_write_uncertain", "{point:?}"); + } + Fault::CancelAfterCommit => { + tokio::pin!(invoke); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + tokio::select! { + _ = adapter.committed.notified() => {} + result = &mut invoke => panic!("expected committed blocked recovery write: {result:?}"), + } + }).await.expect("recovery write must commit before cancellation"); + } + } + } + assert!(adapter.fault.lock().unwrap().is_none()); + let retained: serde_json::Value = + serde_json::from_str(&store.read_maintenance().await.unwrap().unwrap().0).unwrap(); + let retained_invocation = if point == Point::MaintenanceSettlement { + assert_eq!( + retained["outstanding"]["request"]["operation_id"], + request.operation_id + ); + &retained["outstanding"] + } else { + assert!(retained["outstanding"].is_null()); + &retained["results"].as_array().unwrap().last().unwrap()["invocation"] + }; + let settlement = retained_invocation["settlement"].clone(); + assert!(!settlement.is_null()); + let result = + reconcile_maintenance(&recovery, "knowledge", &request.operation_id, &actor) + .await + .unwrap(); + assert_eq!( + serde_json::to_value(&result.invocation.settlement).unwrap(), + settlement + ); + assert!(matches!( + result.completion, + MaintenanceCompletion::Interrupted { + publication: MaintenancePublication::Unpublished + } + )); + let writes = adapter.writes.lock().unwrap().len(); + assert_eq!( + reconcile_maintenance(&recovery, "knowledge", &request.operation_id, &actor) + .await + .unwrap(), + result + ); + assert_eq!( + adapter.writes.lock().unwrap().len(), + writes, + "retained terminal recovery must not write or mint another settlement" + ); + for expected in [ + Point::MaintenanceSettlement, + Point::MaintenanceRecoveryTerminal, + ] { + assert_eq!( + adapter + .writes + .lock() + .unwrap() + .iter() + .filter(|write| Controlled::point( + &write.uri, + &write.payload, + write.expected_version.is_some() + ) == Some(expected)) + .count(), + 1, + "{expected:?} must retain its first committed write" + ); + } + let db = Omnigraph::open(&graph_uri).await.unwrap(); + assert_eq!( + db.graph_manifest_version_of(ReadTarget::branch("main")) + .await + .unwrap(), + base_version + 1 + ); + drop(db); + drop(recovery); + adapter.assert_no_forbidden_calls().await; + } + } +} + #[tokio::test] async fn backend_recorded_late_conditional_writes_cannot_replace_claimed_state() { let (_directory, bundle, store, adapter) = fixture(); diff --git a/crates/omnigraph-cluster/src/deployment/execution.rs b/crates/omnigraph-cluster/src/deployment/execution.rs index de4d3d70b..fb0b92ee4 100644 --- a/crates/omnigraph-cluster/src/deployment/execution.rs +++ b/crates/omnigraph-cluster/src/deployment/execution.rs @@ -1815,6 +1815,7 @@ pub(super) async fn execute_captured_deployment_in_store( } } deployment_policies(store, &before, bundle, caller).await?; + crate::maintenance::require_no_maintenance(store).await?; let sequence = before.next_sequence.unwrap(); let id = requested_id.map(str::to_owned).unwrap_or_else(|| { format!( diff --git a/crates/omnigraph-cluster/src/lib.rs b/crates/omnigraph-cluster/src/lib.rs index eeeb5bad6..c9a42e9b5 100644 --- a/crates/omnigraph-cluster/src/lib.rs +++ b/crates/omnigraph-cluster/src/lib.rs @@ -30,6 +30,7 @@ mod deployment; mod diff; mod graph_read; mod lease; +mod maintenance; mod serve; mod state_lock; mod store; @@ -42,7 +43,8 @@ pub mod lease_protocol { pub use crate::lease::{LeaseKeeper, Watch, lease_dir, watch_and_claim}; } pub use admission::{ - ClusterAdmission, ClusterAdmissionPurpose, acquire_cluster_admission, acquire_graph_admission, + ClusterAdmission, ClusterAdmissionPurpose, acquire_cluster_admission, + acquire_cluster_admission_with_binding, acquire_graph_admission, acquire_graph_admission_with_binding, canonical_graph_uri, }; pub use authorization::{ @@ -56,6 +58,7 @@ use config::{ pub use deployment::*; use diff::{ResourceKind, compute_blast_radius, resource_kind}; pub use graph_read::GraphReadAuthority; +pub use maintenance::*; #[cfg(any(test, feature = "test-util"))] pub use serve::read_serving_snapshot_with_display_root; pub use serve::{ diff --git a/crates/omnigraph-cluster/src/maintenance.rs b/crates/omnigraph-cluster/src/maintenance.rs new file mode 100644 index 000000000..afd8cce7a --- /dev/null +++ b/crates/omnigraph-cluster/src/maintenance.rs @@ -0,0 +1,1930 @@ +//! Durable invocation receipts for maintenance performed by the current native +//! server. This journal is neither graph authority nor another ownership lock. +use omnigraph::db::{ + Omnigraph, OptimizeInput, OptimizeLimits, OptimizePublication, OptimizeReport, + OptimizeSettlement, OptimizeSettlementOutcome, SchemaContractDigest, +}; +use omnigraph_policy::PolicyAction; +use serde::{Deserialize, Serialize}; +use ulid::Ulid; + +use crate::authorization::AppliedPolicies; +use crate::{ClusterAdmission, ClusterState, ClusterStore, Diagnostic, IdentityAuthorization}; + +pub(crate) const MAINTENANCE_FILE: &str = "__cluster/maintenance.json"; +pub(crate) const MAX_JOURNAL_BYTES: usize = 8 * 1024 * 1024; +const MAX_RESULT_BYTES: usize = 1024 * 1024; +const MAX_RESULTS_BYTES: usize = 4 * 1024 * 1024; +const MAX_RESULTS: usize = 32; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MaintenanceKind { + Optimize, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MaintenanceRequest { + pub operation_id: String, + pub graph: String, + pub kind: MaintenanceKind, + pub limits: OptimizeLimits, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MaintenanceStatus { + pub ledger_id: String, + pub next_sequence: u64, + /// Only the requested graph's outstanding identity is exposed. + pub outstanding_id: Option, +} + +impl MaintenanceStatus { + pub fn next_operation_id(&self) -> String { + format!("{}:{}:{}", self.ledger_id, self.next_sequence, Ulid::new()) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MaintenanceInvocation { + pub request: MaintenanceRequest, + pub actor: String, + pub canonical_root: String, + pub result_revision: u64, + /// Admitted applied configuration contract. The actual accepted graph + /// schema and table pins are bound separately by input.before_commit. + pub schema_contract: SchemaContractDigest, + /// Captured under the engine's retained graph gates before any effect. + pub input: Option, + /// Exact native publication identity and completed table effects, retained + /// before its only possible graph publication. + pub publication: Option, + /// Native neutral competitor, persisted before recovery may publish it. + pub settlement: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "status", rename_all = "snake_case", deny_unknown_fields)] +pub enum MaintenancePublication { + Unpublished, + /// The exact table publication is proven. Postpublication derived work + /// and the original full successful completion are not asserted. + Published { + report: OptimizeReport, + }, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "outcome", rename_all = "snake_case", deny_unknown_fields)] +pub enum MaintenanceCompletion { + Success { + report: OptimizeReport, + }, + /// Only a proven pre-effect refusal may close an accepted operation. + Refused { + code: String, + message: String, + }, + Interrupted { + publication: MaintenancePublication, + }, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct MaintenanceResult { + pub operation_id: String, + pub graph: String, + pub invocation: MaintenanceInvocation, + #[serde(flatten)] + pub completion: MaintenanceCompletion, +} + +#[allow(clippy::large_enum_variant)] +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "status", rename_all = "snake_case")] +pub enum MaintenanceLookup { + Outstanding { + operation_id: String, + graph: String, + invocation: MaintenanceInvocation, + }, + Complete { + result: MaintenanceResult, + }, + /// The sequence was consumed. Missing retained evidence establishes neither + /// acceptance of this nonce nor its outcome, and never permits replay. + ResultExpired { + operation_id: String, + }, + NotRecorded { + operation_id: String, + }, + IdentityMismatch { + operation_id: String, + }, + DifferentLedger { + operation_id: String, + }, +} + +/// This value cannot be constructed from a stored receipt or cloned. Keeping +/// its admission alive also prevents a clean owner release during execution. +#[derive(Debug)] +pub struct AcceptedMaintenance { + admission: ClusterAdmission, + invocation: MaintenanceInvocation, +} + +impl AcceptedMaintenance { + pub fn request(&self) -> &MaintenanceRequest { + &self.invocation.request + } +} + +#[allow(clippy::large_enum_variant)] +#[derive(Debug)] +pub enum MaintenanceAcceptance { + Accepted(AcceptedMaintenance), + Existing(MaintenanceLookup), +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Journal { + version: u32, + ledger_id: String, + next_sequence: u64, + outstanding: Option, + results: Vec, +} + +struct OperationId<'a> { + ledger: &'a str, + sequence: u64, +} + +impl<'a> OperationId<'a> { + fn parse(value: &'a str) -> Result { + let invalid = || refusal("maintenance_id_invalid", "invalid maintenance identity"); + if value.len() > 75 { + return Err(invalid()); + } + let mut parts = value.split(':'); + let (Some(ledger), Some(sequence), Some(nonce), None) = + (parts.next(), parts.next(), parts.next(), parts.next()) + else { + return Err(invalid()); + }; + let sequence = sequence + .parse::() + .ok() + .filter(|n| *n > 0 && n.to_string() == value.split(':').nth(1).unwrap_or_default()) + .ok_or_else(invalid)?; + if !canonical_ulid(ledger) || !canonical_ulid(nonce) { + return Err(invalid()); + } + Ok(Self { ledger, sequence }) + } +} + +fn canonical_ulid(value: &str) -> bool { + value + .parse::() + .is_ok_and(|id| id.to_string() == value) +} + +pub(crate) fn refusal(code: &str, message: &str) -> Diagnostic { + Diagnostic::error(code, MAINTENANCE_FILE, message) +} + +fn encode(value: &impl Serialize) -> Result { + serde_json::to_string(value) + .map_err(|_| refusal("maintenance_encode", "could not encode maintenance record")) +} + +fn validate_request(request: &MaintenanceRequest) -> Result<(), Diagnostic> { + OperationId::parse(&request.operation_id)?; + if request.graph.is_empty() + || request.graph.len() > 256 + || request + .graph + .chars() + .any(|c| c.is_control() || matches!(c, '/' | '\\')) + || request.graph == "." + || request.graph == ".." + { + return Err(refusal( + "maintenance_graph_invalid", + "invalid graph identifier", + )); + } + let limits = &request.limits; + if limits.max_tables == 0 + || limits.max_fragments == 0 + || limits.max_rows == 0 + || limits.max_source_bytes == 0 + || limits.max_files == 0 + { + return Err(refusal( + "maintenance_limits_invalid", + "every optimization limit must be positive", + )); + } + Ok(()) +} + +impl Journal { + fn empty(state: &ClusterState) -> Result { + let ledger_id = state + .ledger_id + .clone() + .filter(|id| canonical_ulid(id)) + .ok_or_else(|| { + refusal( + "maintenance_ledger_required", + "maintenance requires a native v2 ledger", + ) + })?; + Ok(Self { + version: 1, + ledger_id, + next_sequence: 1, + outstanding: None, + results: vec![], + }) + } + + fn validate(&self) -> Result<(), Diagnostic> { + let invalid = || { + refusal( + "maintenance_journal_invalid", + "invalid maintenance journal; preserve it for recovery", + ) + }; + if self.version != 1 + || !canonical_ulid(&self.ledger_id) + || self.next_sequence == 0 + || self.results.len() > MAX_RESULTS + || encode(self)?.len() > MAX_JOURNAL_BYTES + { + return Err(invalid()); + } + let mut seen = std::collections::BTreeSet::new(); + let mut total = 0usize; + for entry in self + .results + .iter() + .map(|result| &result.invocation) + .chain(self.outstanding.iter()) + { + validate_request(&entry.request)?; + IdentityAuthorization::authenticated(&entry.actor)?; + if let Some(input) = &entry.input { + validate_input(input, &entry.request.limits)?; + } + if let Some(publication) = &entry.publication { + validate_publication(entry, publication)?; + } + if entry.settlement.is_some() && entry.publication.is_none() { + return Err(invalid()); + } + let id = OperationId::parse(&entry.request.operation_id)?; + if id.ledger != self.ledger_id + || id.sequence >= self.next_sequence + || !seen.insert(id.sequence) + || entry.canonical_root.is_empty() + { + return Err(invalid()); + } + } + if let Some(outstanding) = &self.outstanding { + if OperationId::parse(&outstanding.request.operation_id)?.sequence + != self.next_sequence - 1 + { + return Err(invalid()); + } + } + for result in &self.results { + if result.operation_id != result.invocation.request.operation_id + || result.graph != result.invocation.request.graph + { + return Err(invalid()); + } + validate_completion(&result.completion)?; + validate_success_input(&result.invocation, &result.completion)?; + let bytes = encode(result)?.len(); + if bytes > MAX_RESULT_BYTES { + return Err(invalid()); + } + total = total.checked_add(bytes).ok_or_else(invalid)?; + } + if total > MAX_RESULTS_BYTES { + return Err(invalid()); + } + Ok(()) + } + + fn record(&self, id: &str) -> Option<&MaintenanceInvocation> { + self.outstanding + .as_ref() + .filter(|entry| entry.request.operation_id == id) + .or_else(|| { + self.results + .iter() + .find(|entry| entry.operation_id == id) + .map(|entry| &entry.invocation) + }) + } + + fn lookup(&self, id: &str) -> Result { + let parsed = OperationId::parse(id)?; + let operation_id = id.to_string(); + if parsed.ledger != self.ledger_id { + return Ok(MaintenanceLookup::DifferentLedger { operation_id }); + } + if let Some(entry) = &self.outstanding { + if entry.request.operation_id == id { + return Ok(MaintenanceLookup::Outstanding { + operation_id, + graph: entry.request.graph.clone(), + invocation: entry.clone(), + }); + } + } + if let Some(result) = self.results.iter().find(|entry| entry.operation_id == id) { + return Ok(MaintenanceLookup::Complete { + result: result.clone(), + }); + } + let occupied = self + .results + .iter() + .map(|entry| &entry.operation_id) + .chain( + self.outstanding + .iter() + .map(|entry| &entry.request.operation_id), + ) + .any(|entry| { + OperationId::parse(entry).is_ok_and(|found| found.sequence == parsed.sequence) + }); + Ok(if occupied { + MaintenanceLookup::IdentityMismatch { operation_id } + } else if parsed.sequence < self.next_sequence { + MaintenanceLookup::ResultExpired { operation_id } + } else { + MaintenanceLookup::NotRecorded { operation_id } + }) + } +} + +async fn load( + store: &ClusterStore, + state: &ClusterState, +) -> Result<(Journal, Option), Diagnostic> { + match store.read_maintenance().await? { + None => Ok((Journal::empty(state)?, None)), + Some((text, version)) => { + let journal: Journal = serde_json::from_str(&text).map_err(|_| { + refusal( + "maintenance_journal_invalid", + "could not decode maintenance journal", + ) + })?; + journal.validate()?; + if Some(journal.ledger_id.as_str()) != state.ledger_id.as_deref() { + return Err(refusal( + "maintenance_ledger_mismatch", + "maintenance journal belongs to another native ledger", + )); + } + let root = store.canonical_root()?; + if journal + .results + .iter() + .map(|entry| &entry.invocation) + .chain(journal.outstanding.iter()) + .any(|entry| entry.canonical_root != root) + { + return Err(refusal( + "maintenance_root_mismatch", + "maintenance journal belongs to another canonical root", + )); + } + Ok((journal, Some(version))) + } + } +} + +async fn state(admission: &ClusterAdmission) -> Result<(ClusterStore, ClusterState), Diagnostic> { + admission.validate_serving()?; + admission.validate_deployment().await?; + let store = admission.store(); + let state = store + .read_state(&mut store.observations()) + .await? + .state + .ok_or_else(|| refusal("maintenance_ledger_required", "native ledger is missing"))?; + if state.version != 2 { + return Err(refusal( + "maintenance_ledger_required", + "maintenance requires a native v2 ledger", + )); + } + Ok((store, state)) +} + +async fn authorize( + store: &ClusterStore, + state: &ClusterState, + graph: &str, + identity: &IdentityAuthorization, +) -> Result<(), Diagnostic> { + if !state + .applied_revision + .schema_contracts + .as_ref() + .is_some_and(|contracts| contracts.contains_key(graph)) + { + return Err(refusal( + "graph_not_applied", + "graph is not in the applied native inventory", + )); + } + AppliedPolicies::load_optional(store, state) + .await? + .check_graph(identity.actor(), graph, PolicyAction::Optimize) +} + +/// Read-only status never creates a journal or reserves an operation ID. +pub async fn maintenance_status( + admission: &ClusterAdmission, + graph: &str, + identity: &IdentityAuthorization, +) -> Result { + let (store, state) = state(admission).await?; + authorize(&store, &state, graph, identity).await?; + let (journal, _) = load(&store, &state).await?; + Ok(MaintenanceStatus { + ledger_id: journal.ledger_id, + next_sequence: journal.next_sequence, + outstanding_id: journal + .outstanding + .filter(|entry| entry.request.graph == graph) + .map(|entry| entry.request.operation_id), + }) +} + +/// The original initiator may read only this retained receipt after revocation. +/// Unknown/expired identities require current permission on the requested graph. +pub async fn maintenance_lookup( + admission: &ClusterAdmission, + graph: &str, + operation_id: &str, + identity: &IdentityAuthorization, +) -> Result { + OperationId::parse(operation_id)?; + let (store, state) = state(admission).await?; + let (journal, _) = load(&store, &state).await?; + if let Some(record) = journal.record(operation_id) { + if record.request.graph != graph { + return Err(refusal( + "maintenance_graph_mismatch", + "operation does not match the requested graph", + )); + } + if record.actor != identity.actor() { + authorize(&store, &state, graph, identity).await?; + } + } else { + authorize(&store, &state, graph, identity).await?; + } + journal.lookup(operation_id) +} + +async fn recorded_request( + store: &ClusterStore, + state: &ClusterState, + journal: &Journal, + request: &MaintenanceRequest, + identity: &IdentityAuthorization, +) -> Result, Diagnostic> { + if let Some(record) = journal.record(&request.operation_id) { + if record.actor != identity.actor() || record.request != *request { + return Err(refusal( + "maintenance_input_mismatch", + "operation identity has different immutable input or principal", + )); + } + return journal.lookup(&request.operation_id).map(Some); + } + authorize(store, state, &request.graph, identity).await?; + let lookup = journal.lookup(&request.operation_id)?; + Ok((!matches!(lookup, MaintenanceLookup::NotRecorded { .. })).then_some(lookup)) +} + +/// Compare an original request without reserving or executing anything. A +/// missing identity must still pass ordinary admission under the process gate. +/// Retained duplicates may be read while their operation owns that gate. +pub async fn lookup_maintenance_request( + admission: &ClusterAdmission, + request: &MaintenanceRequest, + identity: &IdentityAuthorization, +) -> Result, Diagnostic> { + validate_request(request)?; + let (store, state) = state(admission).await?; + let (journal, _) = load(&store, &state).await?; + recorded_request(&store, &state, &journal, request, identity).await +} + +/// Must run under the server's existing deployment/maintenance admission gate. +/// A returned Existing value never confers execution permission. +pub async fn accept_maintenance( + admission: &ClusterAdmission, + request: MaintenanceRequest, + identity: &IdentityAuthorization, +) -> Result { + validate_request(&request)?; + let (store, state) = state(admission).await?; + let (mut journal, version) = load(&store, &state).await?; + if let Some(existing) = recorded_request(&store, &state, &journal, &request, identity).await? { + return Ok(MaintenanceAcceptance::Existing(existing)); + } + let id = OperationId::parse(&request.operation_id)?; + if id.sequence != journal.next_sequence { + return Err(refusal( + "maintenance_id_stale", + "new maintenance must use the exact next sequence", + )); + } + if state.outstanding.is_some() { + return Err(refusal( + "cluster_deployment_outstanding", + "native deployment remains outstanding", + )); + } + if journal.outstanding.is_some() { + return Err(refusal( + "cluster_maintenance_outstanding", + "original maintenance remains outstanding", + )); + } + crate::authorization::refuse_pending_recovery(&store).await?; + let schema_contract = state + .applied_revision + .schema_contracts + .as_ref() + .and_then(|contracts| contracts.get(&request.graph)) + .cloned() + .ok_or_else(|| refusal("graph_not_applied", "graph is not applied"))?; + let invocation = MaintenanceInvocation { + request, + actor: identity.actor().to_string(), + canonical_root: admission.canonical_root().to_string(), + result_revision: state + .applied_revision + .result_revision + .ok_or_else(|| refusal("maintenance_ledger_required", "applied revision is missing"))?, + schema_contract, + input: None, + publication: None, + settlement: None, + }; + journal.next_sequence = journal.next_sequence.checked_add(1).ok_or_else(|| { + refusal( + "maintenance_sequence_exhausted", + "maintenance sequence is exhausted", + ) + })?; + journal.outstanding = Some(invocation.clone()); + journal.validate()?; + store + .write_maintenance(&encode(&journal)?, version.as_deref()) + .await?; + Ok(MaintenanceAcceptance::Accepted(AcceptedMaintenance { + admission: admission.clone(), + invocation, + })) +} + +fn validate_completion(completion: &MaintenanceCompletion) -> Result<(), Diagnostic> { + if let MaintenanceCompletion::Refused { code, message } = completion { + if code.is_empty() + || code.len() > 64 + || message.len() > 1024 + || code.chars().any(|c| !c.is_ascii_alphanumeric() && c != '_') + { + return Err(refusal( + "maintenance_result_invalid", + "refusal code or message exceeds its bound", + )); + } + } + Ok(()) +} + +fn validate_input(input: &OptimizeInput, limits: &OptimizeLimits) -> Result<(), Diagnostic> { + if input.before_commit.is_empty() + || input.before_commit.len() > 256 + || input.source_tables > limits.max_tables + || input.source_fragments > limits.max_fragments + || input.source_rows > limits.max_rows + || input.source_bytes > limits.max_source_bytes + || input.source_files > limits.max_files + || input.report_max_bytes == 0 + { + return Err(refusal( + "maintenance_input_limit", + "captured native input exceeds its accepted bounds", + )); + } + Ok(()) +} + +fn validate_success_input( + invocation: &MaintenanceInvocation, + completion: &MaintenanceCompletion, +) -> Result<(), Diagnostic> { + if let MaintenanceCompletion::Success { report } + | MaintenanceCompletion::Interrupted { + publication: MaintenancePublication::Published { report }, + } = completion + { + let matches = invocation.input.as_ref().is_some_and(|input| { + input.before_commit == report.before_commit + && input.source_tables == report.source_tables + && input.source_fragments == report.source_fragments + && input.source_rows == report.source_rows + && input.source_bytes == report.source_bytes + && input.source_files == report.source_files + }); + if !matches { + return Err(refusal( + "maintenance_input_mismatch", + "native completion does not match the durably captured input", + )); + } + if encode(report)?.len() as u64 + > invocation + .input + .as_ref() + .expect("matched input") + .report_max_bytes + { + return Err(refusal( + "maintenance_result_limit", + "native report exceeds its pre-effect reservation", + )); + } + } + Ok(()) +} + +/// Reserve the largest retained envelope before physical effects. A recovery +/// result carries the prepared report as well as its published result; both +/// copies are included. Fixed native identities use their maximal encodings. +fn reserve_result_capacity(invocation: &MaintenanceInvocation) -> Result<(), Diagnostic> { + let input = invocation + .input + .as_ref() + .expect("reservation follows input binding"); + let witness = serde_json::json!({ + "before_manifest_version": u64::MAX, "intent_nonce": "Z".repeat(26), + "schema_contract": input.schema_contract, "actor": invocation.actor, + "created_at": i64::MIN, "report": null, + }); + let settlement = serde_json::json!({"original_digest":"f".repeat(64),"lineage":{ + "graph_commit_id":"Z".repeat(26),"branch":null,"actor_id":"\\".repeat(256), + "merged_parent_commit_id":null,"created_at":i64::MIN, + }}); + let mut envelope = serde_json::json!({ + "operation_id":invocation.request.operation_id,"graph":invocation.request.graph, + "invocation":invocation,"outcome":"interrupted", + "publication":{"status":"published","report":null}, + }); + envelope["invocation"]["publication"] = witness; + envelope["invocation"]["settlement"] = settlement; + let bytes = (encode(&envelope)?.len() as u64) + .checked_sub(8) + .and_then(|n| { + input + .report_max_bytes + .checked_mul(2) + .and_then(|reports| n.checked_add(reports)) + }); + // Refusals can contain escaped diagnostic bytes, whose JSON spelling is + // wider than the raw 1 KiB message bound. Reserve that terminal path too. + let refusal_envelope = serde_json::json!({ + "operation_id":invocation.request.operation_id,"graph":invocation.request.graph, + "invocation":invocation,"outcome":"refused","code":"a".repeat(64), + "message":"\0".repeat(1024), + }); + if bytes.is_none_or(|bytes| bytes > MAX_RESULT_BYTES as u64) + || encode(&refusal_envelope)?.len() > MAX_RESULT_BYTES + { + return Err(refusal( + "maintenance_result_capacity", + "exact optimization and recovery results exceed the retention capacity before effects", + )); + } + Ok(()) +} + +fn validate_publication( + invocation: &MaintenanceInvocation, + publication: &OptimizePublication, +) -> Result<(), Diagnostic> { + validate_success_input( + invocation, + &MaintenanceCompletion::Success { + report: publication.report.clone(), + }, + )?; + if publication.before_manifest_version == 0 + || publication.before_manifest_version == u64::MAX + || !canonical_ulid(&publication.intent_nonce) + || publication.actor.as_deref() != Some(&invocation.actor) + || invocation + .input + .as_ref() + .is_none_or(|input| publication.schema_contract != input.schema_contract) + || publication.report.before_commit != publication.report.after_commit + { + return Err(refusal( + "maintenance_publication_invalid", + "native publication witness does not match its accepted invocation", + )); + } + Ok(()) +} + +fn validate_owner( + admission: &ClusterAdmission, + accepted: &AcceptedMaintenance, +) -> Result<(), Diagnostic> { + if admission.lock_id() != accepted.admission.lock_id() + || admission.canonical_root() != accepted.admission.canonical_root() + { + return Err(refusal( + "maintenance_owner_mismatch", + "operation belongs to another serving owner", + )); + } + Ok(()) +} + +fn validate_binding( + state: &ClusterState, + accepted: &AcceptedMaintenance, +) -> Result<(), Diagnostic> { + if state.outstanding.is_some() + || state.applied_revision.result_revision != Some(accepted.invocation.result_revision) + || state + .applied_revision + .schema_contracts + .as_ref() + .and_then(|contracts| contracts.get(&accepted.request().graph)) + != Some(&accepted.invocation.schema_contract) + { + return Err(refusal( + "maintenance_binding_changed", + "accepted applied configuration changed before completion", + )); + } + Ok(()) +} + +/// Called once by engine preflight while its exact snapshot and native gates +/// remain held. A failed write leaves uncertainty; it is not a safe refusal. +pub async fn bind_maintenance_input( + admission: &ClusterAdmission, + accepted: &mut AcceptedMaintenance, + input: OptimizeInput, +) -> Result<(), Diagnostic> { + validate_owner(admission, accepted)?; + validate_input(&input, &accepted.request().limits)?; + if accepted.invocation.input.is_some() { + return Err(refusal( + "maintenance_input_already_bound", + "native input is already bound to this operation", + )); + } + let (store, state) = state(admission).await?; + validate_binding(&state, accepted)?; + let (mut journal, version) = load(&store, &state).await?; + if journal.outstanding.as_ref() != Some(&accepted.invocation) { + return Err(refusal( + "maintenance_identity_mismatch", + "original accepted maintenance is no longer outstanding", + )); + } + let mut invocation = accepted.invocation.clone(); + invocation.input = Some(input); + reserve_result_capacity(&invocation)?; + journal.outstanding = Some(invocation.clone()); + journal.validate()?; + store + .write_maintenance(&encode(&journal)?, version.as_deref()) + .await?; + accepted.invocation = invocation; + Ok(()) +} + +/// Native Core invokes this while retaining the original graph gates and +/// before its exact-version publication. Failure never permits publication. +pub async fn bind_maintenance_publication( + admission: &ClusterAdmission, + accepted: &mut AcceptedMaintenance, + publication: OptimizePublication, +) -> Result<(), Diagnostic> { + validate_owner(admission, accepted)?; + validate_publication(&accepted.invocation, &publication)?; + if accepted.invocation.publication.is_some() { + return Err(refusal( + "maintenance_publication_already_bound", + "native publication is already bound", + )); + } + let (store, state) = state(admission).await?; + validate_binding(&state, accepted)?; + let (mut journal, version) = load(&store, &state).await?; + if journal.outstanding.as_ref() != Some(&accepted.invocation) { + return Err(refusal( + "maintenance_identity_mismatch", + "original maintenance is no longer outstanding", + )); + } + let mut invocation = accepted.invocation.clone(); + invocation.publication = Some(publication); + journal.outstanding = Some(invocation.clone()); + journal.validate()?; + store + .write_maintenance(&encode(&journal)?, version.as_deref()) + .await?; + accepted.invocation = invocation; + Ok(()) +} + +/// Persist the exact native completion once. Failure leaves the original +/// outstanding identity unresolved; it never grants another execution token. +pub async fn complete_maintenance( + admission: &ClusterAdmission, + accepted: AcceptedMaintenance, + completion: MaintenanceCompletion, +) -> Result { + validate_completion(&completion)?; + validate_owner(admission, &accepted)?; + validate_success_input(&accepted.invocation, &completion)?; + let (store, state) = state(admission).await?; + validate_binding(&state, &accepted)?; + let (mut journal, version) = load(&store, &state).await?; + if journal.outstanding.as_ref() != Some(&accepted.invocation) { + return Err(refusal( + "maintenance_identity_mismatch", + "original accepted maintenance is no longer outstanding", + )); + } + let result = MaintenanceResult { + operation_id: accepted.request().operation_id.clone(), + graph: accepted.request().graph.clone(), + invocation: accepted.invocation, + completion, + }; + append_result(&mut journal, &result)?; + store + .write_maintenance(&encode(&journal)?, version.as_deref()) + .await?; + Ok(result) +} + +fn append_result(journal: &mut Journal, result: &MaintenanceResult) -> Result<(), Diagnostic> { + if encode(result)?.len() > MAX_RESULT_BYTES { + return Err(refusal( + "maintenance_result_limit", + "exact maintenance result exceeds its retention byte limit", + )); + } + journal.results.push(result.clone()); + while journal.results.len() > MAX_RESULTS + || journal + .results + .iter() + .map(encode) + .collect::, _>>()? + .iter() + .map(String::len) + .sum::() + > MAX_RESULTS_BYTES + { + journal.results.remove(0); + } + journal.outstanding = None; + journal.validate()?; + Ok(()) +} + +pub(crate) async fn require_maintenance_recovery( + store: &ClusterStore, + state: &ClusterState, + operation_id: &str, +) -> Result<(), Diagnostic> { + OperationId::parse(operation_id)?; + let (journal, _) = load(store, state).await?; + if journal + .outstanding + .as_ref() + .is_some_and(|entry| entry.request.operation_id != operation_id) + || journal.record(operation_id).is_none() + { + return Err(refusal( + "maintenance_recovery_identity_mismatch", + "only the exact retained original maintenance identity may recover", + )); + } + Ok(()) +} + +/// Reconcile an original operation under recovery-only native admission. +/// The caller first excludes predecessor processes, other writers and cleanup, +/// then handles the exact prior lock using the existing administrator procedure. +/// No physical optimization is rerun. Errors retain the journal and native lock. +pub async fn reconcile_maintenance( + admission: &ClusterAdmission, + graph: &str, + operation_id: &str, + identity: &IdentityAuthorization, +) -> Result { + admission + .validate_maintenance_recovery(operation_id) + .await?; + let store = admission.store(); + let state = store + .read_state(&mut store.observations()) + .await? + .state + .ok_or_else(|| refusal("maintenance_ledger_required", "native ledger is missing"))?; + let (mut journal, mut version) = load(&store, &state).await?; + let record = journal.record(operation_id).ok_or_else(|| { + refusal( + "maintenance_recovery_identity_mismatch", + "original operation is not retained", + ) + })?; + if record.request.graph != graph { + return Err(refusal( + "maintenance_graph_mismatch", + "operation does not match requested graph", + )); + } + authorize(&store, &state, graph, identity).await?; + if let Some(result) = journal + .results + .iter() + .find(|entry| entry.operation_id == operation_id) + { + return Ok(result.clone()); + } + let mut invocation = journal + .outstanding + .clone() + .filter(|entry| entry.request.operation_id == operation_id) + .ok_or_else(|| { + refusal( + "maintenance_recovery_identity_mismatch", + "original operation is not outstanding", + ) + })?; + let accepted = AcceptedMaintenance { + admission: admission.clone(), + invocation: invocation.clone(), + }; + validate_binding(&state, &accepted)?; + drop(accepted); + let publication = match invocation.publication.clone() { + None => MaintenancePublication::Unpublished, + Some(prepared) => { + let policies = AppliedPolicies::load_optional(&store, &state).await?; + let uri = store.graph_root(graph); + Omnigraph::ensure_no_pending_recovery_with_storage(&uri, store.graph_storage(&uri)?) + .await + .map_err(|error| refusal("graph_recovery_required", &error.to_string()))?; + let db = Omnigraph::open_with_storage(&uri, store.graph_storage(&uri)?) + .await + .map_err(|error| refusal("maintenance_recovery_failed", &error.to_string()))?; + let db = match policies.graph(graph) { + Some(policy) => db.with_policy(policy), + None => db, + }; + if invocation.settlement.is_none() { + invocation.settlement = Some( + db.prepare_optimize_settlement_as(&prepared, Some(identity.actor())) + .await + .map_err(|error| { + refusal("maintenance_recovery_failed", &error.to_string()) + })?, + ); + journal.outstanding = Some(invocation.clone()); + journal.validate()?; + store + .write_maintenance(&encode(&journal)?, version.as_deref()) + .await?; + // Reread the committed exact bytes, never guess the CAS token. + let (written, next_version) = load(&store, &state).await?; + if written.outstanding.as_ref() != Some(&invocation) { + return Err(refusal( + "maintenance_identity_mismatch", + "recovery binding changed", + )); + } + journal = written; + version = next_version; + } + admission + .validate_maintenance_recovery(operation_id) + .await?; + match db + .settle_optimize_as( + &prepared, + invocation + .settlement + .as_ref() + .expect("persisted settlement"), + Some(identity.actor()), + ) + .await + .map_err(|error| refusal("maintenance_recovery_failed", &error.to_string()))? + { + OptimizeSettlementOutcome::Published { report } => { + MaintenancePublication::Published { report } + } + OptimizeSettlementOutcome::Unpublished => MaintenancePublication::Unpublished, + OptimizeSettlementOutcome::Unknown => { + return Err(refusal( + "maintenance_recovery_unknown", + "original publication remains unknown; retain original identity and ownership", + )); + } + } + } + }; + admission + .validate_maintenance_recovery(operation_id) + .await?; + let current = store + .read_state(&mut store.observations()) + .await? + .state + .ok_or_else(|| refusal("maintenance_ledger_required", "native ledger is missing"))?; + let accepted = AcceptedMaintenance { + admission: admission.clone(), + invocation: invocation.clone(), + }; + validate_binding(¤t, &accepted)?; + drop(accepted); + let result = MaintenanceResult { + operation_id: operation_id.to_string(), + graph: graph.to_string(), + invocation, + completion: MaintenanceCompletion::Interrupted { publication }, + }; + append_result(&mut journal, &result)?; + store + .write_maintenance(&encode(&journal)?, version.as_deref()) + .await?; + Ok(result) +} + +/// Deployment and clean release must not erase unresolved maintenance evidence. +/// This check is observational; the owning server also serializes acceptance. +pub(crate) async fn require_no_maintenance(store: &ClusterStore) -> Result<(), Diagnostic> { + let Some((text, _)) = store.read_maintenance().await? else { + return Ok(()); + }; + let journal: Journal = serde_json::from_str(&text).map_err(|_| { + refusal( + "maintenance_journal_invalid", + "could not decode maintenance journal", + ) + })?; + journal.validate()?; + if journal.outstanding.is_some() { + return Err(refusal( + "cluster_maintenance_outstanding", + "original maintenance remains outstanding; ownership must be retained", + )); + } + Ok(()) +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use crate::{ + ClusterAdmissionPurpose, DeploymentCaller, DeploymentLookup, acquire_cluster_admission, + apply_deployment, deployment_status, force_unlock_storage_root, + }; + use std::fs; + + fn identity(actor: &str) -> IdentityAuthorization { + IdentityAuthorization::authenticated(actor).unwrap() + } + + fn limits() -> OptimizeLimits { + OptimizeLimits { + max_tables: 8, + max_fragments: 64, + max_rows: 1000, + max_source_bytes: 1024 * 1024, + max_files: 128, + } + } + + pub(crate) async fn fixture() -> (tempfile::TempDir, ClusterAdmission) { + let dir = crate::tests::identity_fixture(); + let policy = dir.path().join("base.policy.yaml"); + let source = fs::read_to_string(&policy).unwrap(); + fs::write(policy, format!("{source}\n - id: owners-optimize\n allow: {{actors: {{group: owners}}, actions: [optimize], target_branch_scope: protected}}\nprotected_branches: [main]\n")).unwrap(); + let caller = DeploymentCaller::storage_owner(None); + let result = apply_deployment(dir.path(), None, &caller, |_, _, _| {}) + .await + .unwrap(); + assert!(matches!(result, DeploymentLookup::Complete { ref result } if result.converged)); + let root = dir.path().to_str().unwrap(); + if let Some(lock) = deployment_status(root, None, &caller) + .await + .unwrap() + .lock_id + { + force_unlock_storage_root(root, &lock).await.unwrap(); + } + let admission = acquire_cluster_admission(root, ClusterAdmissionPurpose::Serve) + .await + .unwrap() + .unwrap(); + (dir, admission) + } + + pub(crate) async fn request(admission: &ClusterAdmission) -> MaintenanceRequest { + let status = maintenance_status(admission, "knowledge", &identity("principal:owner")) + .await + .unwrap(); + MaintenanceRequest { + operation_id: status.next_operation_id(), + graph: "knowledge".into(), + kind: MaintenanceKind::Optimize, + limits: limits(), + } + } + + fn refusal_completion() -> MaintenanceCompletion { + MaintenanceCompletion::Refused { + code: "optimize_limit_exceeded".into(), + message: "source exceeds the accepted bound before effects".into(), + } + } + + #[tokio::test] + async fn admission_is_durable_permission_based_and_duplicate_never_executes() { + let (dir, admission) = fixture().await; + let request = request(&admission).await; + assert!( + lookup_maintenance_request(&admission, &request, &identity("principal:owner")) + .await + .unwrap() + .is_none() + ); + assert!( + !dir.path().join(MAINTENANCE_FILE).exists(), + "status must not create state" + ); + assert!( + maintenance_status(&admission, "knowledge", &identity("principal:reader")) + .await + .is_err() + ); + assert!( + accept_maintenance(&admission, request.clone(), &identity("principal:reader")) + .await + .is_err() + ); + assert!(!dir.path().join(MAINTENANCE_FILE).exists()); + let MaintenanceAcceptance::Accepted(accepted) = + accept_maintenance(&admission, request.clone(), &identity("principal:owner")) + .await + .unwrap() + else { + panic!("expected execution token") + }; + assert!(dir.path().join(MAINTENANCE_FILE).exists()); + let journal_bytes = fs::read(dir.path().join(MAINTENANCE_FILE)).unwrap(); + assert!(matches!( + lookup_maintenance_request(&admission, &request, &identity("principal:owner")) + .await + .unwrap(), + Some(MaintenanceLookup::Outstanding { .. }) + )); + assert_eq!( + fs::read(dir.path().join(MAINTENANCE_FILE)).unwrap(), + journal_bytes + ); + assert!(matches!( + accept_maintenance(&admission, request.clone(), &identity("principal:owner")) + .await + .unwrap(), + MaintenanceAcceptance::Existing(MaintenanceLookup::Outstanding { .. }) + )); + let mut mismatch = request.clone(); + mismatch.limits.max_rows += 1; + assert_eq!( + accept_maintenance(&admission, mismatch, &identity("principal:owner")) + .await + .unwrap_err() + .code, + "maintenance_input_mismatch" + ); + assert_eq!( + accept_maintenance( + &admission, + request.clone(), + &identity("principal:collaborator") + ) + .await + .unwrap_err() + .code, + "maintenance_input_mismatch" + ); + assert_eq!( + require_no_maintenance(&admission.store()) + .await + .unwrap_err() + .code, + "cluster_maintenance_outstanding" + ); + let next = self::request(&admission).await; + assert_eq!( + accept_maintenance(&admission, next, &identity("principal:owner")) + .await + .unwrap_err() + .code, + "cluster_maintenance_outstanding" + ); + let result = complete_maintenance(&admission, accepted, refusal_completion()) + .await + .unwrap(); + assert_eq!(result.operation_id, request.operation_id); + let encoded = serde_json::to_value(&result).unwrap(); + assert_eq!(encoded["outcome"], "refused"); + assert!(encoded.get("report").is_none()); + let decoded: MaintenanceResult = serde_json::from_value(encoded).unwrap(); + assert_eq!(decoded, result); + assert!(matches!( + accept_maintenance(&admission, request, &identity("principal:owner")) + .await + .unwrap(), + MaintenanceAcceptance::Existing(MaintenanceLookup::Complete { .. }) + )); + require_no_maintenance(&admission.store()).await.unwrap(); + // Production shutdown closes mutation admission before native release; + // the journal blocker check is a read and must remain possible. + admission.io_scope().unwrap().close(); + admission.release_after_settlement().await.unwrap(); + } + + #[tokio::test] + async fn abandoned_acceptance_blocks_release_and_original_lookup_is_retained() { + let (dir, admission) = fixture().await; + let request = request(&admission).await; + let MaintenanceAcceptance::Accepted(token) = + accept_maintenance(&admission, request.clone(), &identity("principal:owner")) + .await + .unwrap() + else { + panic!() + }; + drop(token); + let bundle = crate::capture_deployment(dir.path()).unwrap(); + let mut effects_started = false; + let deployment = crate::apply_captured_deployment( + &bundle, + None, + &DeploymentCaller::storage_owner(None), + &admission, + &std::collections::BTreeMap::new(), + |_, _, _| {}, + |_| {}, + &mut effects_started, + ) + .await + .unwrap_err(); + assert_eq!(deployment.code, "cluster_maintenance_outstanding"); + assert!(!effects_started); + assert!(matches!( + maintenance_lookup( + &admission, + "knowledge", + &request.operation_id, + &identity("principal:owner") + ) + .await + .unwrap(), + MaintenanceLookup::Outstanding { .. } + )); + assert_eq!( + maintenance_lookup( + &admission, + "other", + &request.operation_id, + &identity("principal:owner") + ) + .await + .unwrap_err() + .code, + "maintenance_graph_mismatch" + ); + assert!( + maintenance_lookup( + &admission, + "knowledge", + &request.operation_id, + &identity("principal:reader") + ) + .await + .is_err() + ); + assert_eq!( + admission.release_after_settlement().await.unwrap_err().code, + "cluster_maintenance_outstanding" + ); + // Administrative process exclusion and lock retirement do not settle + // the accepted operation. Every later writable admission must still + // preserve that unresolved identity instead of reopening its graphs. + let root = dir.path().to_str().unwrap(); + for purpose in [ + ClusterAdmissionPurpose::Serve, + ClusterAdmissionPurpose::GraphOperation, + ClusterAdmissionPurpose::Deployment, + ] { + let lock = deployment_status(root, None, &DeploymentCaller::storage_owner(None)) + .await + .unwrap() + .lock_id + .unwrap(); + force_unlock_storage_root(root, &lock).await.unwrap(); + let error = acquire_cluster_admission(root, purpose) + .await + .expect_err("an unresolved optimization must block successor writers"); + assert!( + error + .message + .contains("original maintenance remains outstanding") + ); + } + let lock = deployment_status(root, None, &DeploymentCaller::storage_owner(None)) + .await + .unwrap() + .lock_id + .unwrap(); + force_unlock_storage_root(root, &lock).await.unwrap(); + let recovery = acquire_cluster_admission( + root, + ClusterAdmissionPurpose::MaintenanceRecovery { + operation_id: request.operation_id.clone(), + }, + ) + .await + .unwrap() + .unwrap(); + assert!(recovery.validate_serving().is_err()); + assert!(recovery.validate_deployment().await.is_err()); + assert!( + recovery + .validate_graph_uri(&format!("{root}/graphs/knowledge.omni")) + .await + .is_err() + ); + assert!( + reconcile_maintenance( + &recovery, + "knowledge", + &request.operation_id, + &identity("principal:reader") + ) + .await + .is_err() + ); + assert_eq!( + reconcile_maintenance( + &recovery, + "other", + &request.operation_id, + &identity("principal:owner") + ) + .await + .unwrap_err() + .code, + "maintenance_graph_mismatch" + ); + let result = reconcile_maintenance( + &recovery, + "knowledge", + &request.operation_id, + &identity("principal:owner"), + ) + .await + .unwrap(); + assert!(matches!( + result.completion, + MaintenanceCompletion::Interrupted { + publication: MaintenancePublication::Unpublished + } + )); + assert_eq!(result.invocation.actor, "principal:owner"); + let bytes = fs::read(dir.path().join(MAINTENANCE_FILE)).unwrap(); + let duplicate = reconcile_maintenance( + &recovery, + "knowledge", + &request.operation_id, + &identity("principal:owner"), + ) + .await + .unwrap(); + assert_eq!(duplicate, result); + assert_eq!(fs::read(dir.path().join(MAINTENANCE_FILE)).unwrap(), bytes); + recovery.release_after_settlement().await.unwrap(); + let successor = acquire_cluster_admission(root, ClusterAdmissionPurpose::Serve) + .await + .unwrap() + .unwrap(); + assert!(matches!( + maintenance_lookup( + &successor, + "knowledge", + &request.operation_id, + &identity("principal:owner") + ) + .await + .unwrap(), + MaintenanceLookup::Complete { .. } + )); + successor.release_after_settlement().await.unwrap(); + } + + #[tokio::test] + async fn result_capacity_refuses_before_binding_or_effect_permission() { + fn graph_files( + root: &std::path::Path, + ) -> std::collections::BTreeMap { + let mut pending = vec![root.to_path_buf()]; + let mut files = std::collections::BTreeMap::new(); + while let Some(directory) = pending.pop() { + for entry in fs::read_dir(directory).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + pending.push(path); + } else { + files.insert( + path.strip_prefix(root).unwrap().to_path_buf(), + crate::sha256_hex(&fs::read(path).unwrap()), + ); + } + } + } + files + } + let (dir, admission) = fixture().await; + let store = admission.store(); + let uri = store.graph_root("knowledge"); + let db = Omnigraph::open_with_storage(&uri, store.graph_storage(&uri).unwrap()) + .await + .unwrap(); + // Real accepted metadata, not a fabricated report: the retained table + // name alone makes the original witness plus result exceed one MiB. + // Z sorts after productive Person, proving the entire attempt preflights. + let long_type = "Z".repeat(600 * 1024); + db.apply_schema(&format!( + "{}\nnode {long_type} {{ name: String @key }}\n", + db.schema_source() + )) + .await + .unwrap(); + let db = omnigraph::Session::from_defaults( + std::sync::Arc::new(db), + omnigraph::settings::SessionSettings::default(), + ); + for name in ["Ada", "Grace", "Barbara"] { + db.load_jsonl( + &format!(r#"{{"type":"Person","data":{{"name":"{name}","age":37}}}}"#), + omnigraph::loader::LoadMode::Merge, + ) + .await + .unwrap(); + } + let snapshot = db + .snapshot_of(omnigraph::db::ReadTarget::branch("main")) + .await + .unwrap(); + assert!(snapshot.dataset(&format!("node:{long_type}")).is_some()); + let graph_path = dir.path().join("graphs/knowledge.omni"); + let physical_before = graph_files(&graph_path); + let mut request = request(&admission).await; + request.limits.max_source_bytes = 64 * 1024 * 1024; + let limits = request.limits.clone(); + let MaintenanceAcceptance::Accepted(mut accepted) = + accept_maintenance(&admission, request, &identity("principal:owner")) + .await + .unwrap() + else { + panic!() + }; + let before = fs::read(dir.path().join(MAINTENANCE_FILE)).unwrap(); + let accepted_ref = &mut accepted; + let admission_ref = &admission; + let failure = db + .optimize_with_limits_admitted_as( + &limits, + Some("principal:owner"), + |input| async move { + assert!(input.report_max_bytes * 2 > MAX_RESULT_BYTES as u64); + let error = bind_maintenance_input(admission_ref, accepted_ref, input) + .await + .unwrap_err(); + assert_eq!(error.code, "maintenance_result_capacity"); + Err(omnigraph::error::OmniError::manifest(error.code)) + }, + ) + .await + .unwrap_err(); + assert!(!failure.effects_started); + assert!( + failure + .error + .to_string() + .contains("maintenance_result_capacity") + ); + assert!(accepted.invocation.input.is_none()); + assert_eq!(fs::read(dir.path().join(MAINTENANCE_FILE)).unwrap(), before); + assert_eq!( + graph_files(&graph_path), + physical_before, + "preflight must not write graph files" + ); + let after = db + .snapshot_of(omnigraph::db::ReadTarget::branch("main")) + .await + .unwrap(); + assert_eq!( + after.graph_manifest_version(), + snapshot.graph_manifest_version() + ); + assert_eq!(after.graph_head(None), snapshot.graph_head(None)); + for entry in snapshot.datasets() { + let unchanged = after.dataset(&entry.type_key).unwrap(); + assert_eq!(unchanged.identity, entry.identity); + assert_eq!(unchanged.dataset_path, entry.dataset_path); + assert_eq!( + unchanged.published_dataset_version, + entry.published_dataset_version + ); + assert_eq!(unchanged.native_dataset_branch, entry.native_dataset_branch); + assert_eq!(unchanged.entity_count, entry.entity_count); + } + let result = complete_maintenance( + &admission, + accepted, + MaintenanceCompletion::Refused { + code: "maintenance_result_capacity".into(), + message: "result cannot fit".into(), + }, + ) + .await + .unwrap(); + assert!( + matches!(result.completion, MaintenanceCompletion::Refused { ref code, .. } if code == "maintenance_result_capacity") + ); + require_no_maintenance(&admission.store()).await.unwrap(); + let next = self::request(&admission).await; + let MaintenanceAcceptance::Accepted(next) = + accept_maintenance(&admission, next, &identity("principal:owner")) + .await + .unwrap() + else { + panic!() + }; + complete_maintenance(&admission, next, refusal_completion()) + .await + .unwrap(); + admission.release_after_settlement().await.unwrap(); + } + + #[tokio::test] + async fn exact_input_is_bound_once_and_success_must_report_that_input() { + let (_dir, admission) = fixture().await; + let request = request(&admission).await; + let actor = identity("principal:owner"); + let MaintenanceAcceptance::Accepted(mut accepted) = + accept_maintenance(&admission, request.clone(), &actor) + .await + .unwrap() + else { + panic!() + }; + let input = OptimizeInput { + before_commit: "retained-commit".into(), + report_max_bytes: 16384, + schema_contract: accepted.invocation.schema_contract.clone(), + source_tables: 1, + source_fragments: 2, + source_rows: 5, + source_bytes: 1024, + source_files: 2, + }; + bind_maintenance_input(&admission, &mut accepted, input.clone()) + .await + .unwrap(); + assert_eq!( + bind_maintenance_input(&admission, &mut accepted, input.clone()) + .await + .unwrap_err() + .code, + "maintenance_input_already_bound" + ); + let MaintenanceLookup::Outstanding { invocation, .. } = + maintenance_lookup(&admission, "knowledge", &request.operation_id, &actor) + .await + .unwrap() + else { + panic!() + }; + assert_eq!(invocation.input, Some(input.clone())); + let mut report = OptimizeReport { + before_commit: input.before_commit, + after_commit: "result-commit".into(), + tables: vec![], + source_tables: input.source_tables, + source_fragments: input.source_fragments, + source_rows: input.source_rows, + source_bytes: input.source_bytes, + source_files: input.source_files, + }; + report.source_rows += 1; + assert!( + validate_success_input( + &accepted.invocation, + &MaintenanceCompletion::Success { + report: report.clone() + } + ) + .is_err() + ); + report.source_rows -= 1; + let result = complete_maintenance( + &admission, + accepted, + MaintenanceCompletion::Success { report }, + ) + .await + .unwrap(); + let encoded = serde_json::to_value(&result).unwrap(); + assert_eq!(encoded["outcome"], "success"); + assert_eq!( + serde_json::from_value::(encoded).unwrap(), + result + ); + admission.release_after_settlement().await.unwrap(); + } + + #[tokio::test] + async fn revoked_initiator_can_read_only_its_retained_receipt() { + let (dir, admission) = fixture().await; + let actor = identity("principal:owner"); + let request = request(&admission).await; + let MaintenanceAcceptance::Accepted(accepted) = + accept_maintenance(&admission, request.clone(), &actor) + .await + .unwrap() + else { + panic!() + }; + complete_maintenance(&admission, accepted, refusal_completion()) + .await + .unwrap(); + admission.release_after_settlement().await.unwrap(); + let policy = dir.path().join("base.policy.yaml"); + let source = fs::read_to_string(&policy).unwrap(); + let without_optimize = source.split(" - id: owners-optimize").next().unwrap(); + fs::write(policy, without_optimize).unwrap(); + let caller = DeploymentCaller::storage_owner(None); + let applied = apply_deployment(dir.path(), None, &caller, |_, _, _| {}) + .await + .unwrap(); + assert!(matches!(applied, DeploymentLookup::Complete { ref result } if result.converged)); + let root = dir.path().to_str().unwrap(); + if let Some(lock) = deployment_status(root, None, &caller) + .await + .unwrap() + .lock_id + { + force_unlock_storage_root(root, &lock).await.unwrap(); + } + let admission = acquire_cluster_admission(root, ClusterAdmissionPurpose::Serve) + .await + .unwrap() + .unwrap(); + assert!( + maintenance_status(&admission, "knowledge", &actor) + .await + .is_err() + ); + assert!(matches!( + maintenance_lookup(&admission, "knowledge", &request.operation_id, &actor) + .await + .unwrap(), + MaintenanceLookup::Complete { .. } + )); + assert!( + maintenance_lookup( + &admission, + "knowledge", + &request.operation_id, + &identity("principal:collaborator") + ) + .await + .is_err() + ); + let parsed = OperationId::parse(&request.operation_id).unwrap(); + let unknown = format!("{}:{}:{}", parsed.ledger, parsed.sequence + 1, Ulid::new()); + assert!( + maintenance_lookup(&admission, "knowledge", &unknown, &actor) + .await + .is_err() + ); + admission.release_after_settlement().await.unwrap(); + } + + #[tokio::test] + async fn oversized_exact_result_remains_outstanding_instead_of_truncated_success() { + let (_dir, admission) = fixture().await; + let actor = identity("principal:owner"); + let request = request(&admission).await; + let MaintenanceAcceptance::Accepted(mut accepted) = + accept_maintenance(&admission, request.clone(), &actor) + .await + .unwrap() + else { + panic!() + }; + let input = OptimizeInput { + before_commit: "retained-commit".into(), + report_max_bytes: 16384, + schema_contract: accepted.invocation.schema_contract.clone(), + source_tables: 0, + source_fragments: 0, + source_rows: 0, + source_bytes: 0, + source_files: 0, + }; + bind_maintenance_input(&admission, &mut accepted, input.clone()) + .await + .unwrap(); + let report = OptimizeReport { + before_commit: input.before_commit, + after_commit: "x".repeat(MAX_RESULT_BYTES), + tables: vec![], + source_tables: 0, + source_fragments: 0, + source_rows: 0, + source_bytes: 0, + source_files: 0, + }; + assert_eq!( + complete_maintenance( + &admission, + accepted, + MaintenanceCompletion::Success { report } + ) + .await + .unwrap_err() + .code, + "maintenance_result_limit" + ); + assert!(matches!( + maintenance_lookup(&admission, "knowledge", &request.operation_id, &actor) + .await + .unwrap(), + MaintenanceLookup::Outstanding { .. } + )); + assert_eq!( + admission.release_after_settlement().await.unwrap_err().code, + "cluster_maintenance_outstanding" + ); + } + + #[tokio::test] + async fn retention_expiry_is_never_replay_permission_and_cas_rejects_stale_bytes() { + let (_dir, admission) = fixture().await; + let actor = identity("principal:owner"); + let first = request(&admission).await; + for index in 0..=MAX_RESULTS { + let request = if index == 0 { + first.clone() + } else { + self::request(&admission).await + }; + let MaintenanceAcceptance::Accepted(accepted) = + accept_maintenance(&admission, request, &actor) + .await + .unwrap() + else { + panic!() + }; + complete_maintenance(&admission, accepted, refusal_completion()) + .await + .unwrap(); + } + assert!(matches!( + maintenance_lookup(&admission, "knowledge", &first.operation_id, &actor) + .await + .unwrap(), + MaintenanceLookup::ResultExpired { .. } + )); + assert!(matches!( + accept_maintenance(&admission, first, &actor).await.unwrap(), + MaintenanceAcceptance::Existing(MaintenanceLookup::ResultExpired { .. }) + )); + let current = request(&admission).await; + let mut future = current.clone(); + let parsed = OperationId::parse(&future.operation_id).unwrap(); + future.operation_id = format!("{}:{}:{}", parsed.ledger, parsed.sequence + 1, Ulid::new()); + assert_eq!( + accept_maintenance(&admission, future, &actor) + .await + .unwrap_err() + .code, + "maintenance_id_stale" + ); + let store = admission.store(); + let (previous, version) = store.read_maintenance().await.unwrap().unwrap(); + let MaintenanceAcceptance::Accepted(accepted) = + accept_maintenance(&admission, current.clone(), &actor) + .await + .unwrap() + else { + panic!() + }; + assert_eq!( + store + .write_maintenance(&previous, Some(&version)) + .await + .unwrap_err() + .code, + "maintenance_cas_mismatch" + ); + let mut same_sequence = current.clone(); + let parsed = OperationId::parse(¤t.operation_id).unwrap(); + same_sequence.operation_id = + format!("{}:{}:{}", parsed.ledger, parsed.sequence, Ulid::new()); + assert!(matches!( + accept_maintenance(&admission, same_sequence, &actor) + .await + .unwrap(), + MaintenanceAcceptance::Existing(MaintenanceLookup::IdentityMismatch { .. }) + )); + complete_maintenance(&admission, accepted, refusal_completion()) + .await + .unwrap(); + let (text, _) = store.read_maintenance().await.unwrap().unwrap(); + let journal: Journal = serde_json::from_str(&text).unwrap(); + assert_eq!(journal.results.len(), MAX_RESULTS); + assert!(text.len() < MAX_RESULTS_BYTES); + admission.release_after_settlement().await.unwrap(); + } + + #[test] + fn identities_and_failure_records_are_strict_and_bounded() { + for id in [ + "", + "x:1:x", + "01ARZ3NDEKTSV4RRFFQ69G5FAV:01:01ARZ3NDEKTSV4RRFFQ69G5FAV", + "01ARZ3NDEKTSV4RRFFQ69G5FAV:0:01ARZ3NDEKTSV4RRFFQ69G5FAV", + ] { + assert!(OperationId::parse(id).is_err()); + } + assert!( + validate_completion(&MaintenanceCompletion::Refused { + code: "too_long".repeat(10), + message: "bounded".into() + }) + .is_err() + ); + assert!( + validate_completion(&MaintenanceCompletion::Refused { + code: "valid".into(), + message: "x".repeat(1025) + }) + .is_err() + ); + } +} diff --git a/crates/omnigraph-cluster/src/store.rs b/crates/omnigraph-cluster/src/store.rs index 43cb4ee8b..2398f8753 100644 --- a/crates/omnigraph-cluster/src/store.rs +++ b/crates/omnigraph-cluster/src/store.rs @@ -557,6 +557,51 @@ impl ClusterStore { .map_err(|err| err.to_string()) } + /// Bounded maintenance journal access through this owner's existing I/O + /// scope and captured storage binding. The version is an opaque CAS token. + pub(crate) async fn read_maintenance(&self) -> Result, Diagnostic> { + self.adapter + .read_text_versioned_if_exists_bounded( + &self.uri(crate::maintenance::MAINTENANCE_FILE), + crate::maintenance::MAX_JOURNAL_BYTES as u64, + ) + .await + .map_err(|_| { + crate::maintenance::refusal( + "maintenance_read_error", + "could not read the maintenance journal", + ) + }) + } + + /// A failed or unacknowledged conditional write never authorizes a retry + /// of graph effects. The caller retains its original operation identity. + pub(crate) async fn write_maintenance( + &self, + payload: &str, + expected_version: Option<&str>, + ) -> Result<(), Diagnostic> { + use crate::maintenance::{MAINTENANCE_FILE, MAX_JOURNAL_BYTES, refusal}; + if payload.len() > MAX_JOURNAL_BYTES { + return Err(refusal( + "maintenance_journal_limit", + "maintenance journal exceeds its byte limit", + )); + } + let uri = self.uri(MAINTENANCE_FILE); + let written = match expected_version { + Some(version) => self.adapter.write_text_if_match(&uri, payload, version).await.map(|value| value.is_some()), + None => self.adapter.write_text_if_absent(&uri, payload).await, + }.map_err(|_| refusal("maintenance_write_uncertain", "maintenance journal write was not acknowledged; inspect the original identity without replaying effects"))?; + if !written { + return Err(refusal( + "maintenance_cas_mismatch", + "maintenance journal changed; inspect the original identity without replaying effects", + )); + } + Ok(()) + } + /// Shared list-and-parse for the sidecar/approval directories: id /// (filename) order; unparseable objects warn and stay for the operator. async fn list_json_dir( @@ -1186,6 +1231,7 @@ impl ClusterStore { } pub(crate) async fn release_settled(&self, lock_id: &str) -> Result<(), Diagnostic> { + crate::maintenance::require_no_maintenance(self).await?; // Permanently close graph/control adapters before the final lock write. // This private client can only execute the exact-owner release below; // no caller receives a general escape from a closed storage scope. diff --git a/crates/omnigraph-dst/cost_table.txt b/crates/omnigraph-dst/cost_table.txt index 23ad9f701..074c3dada 100644 --- a/crates/omnigraph-dst/cost_table.txt +++ b/crates/omnigraph-dst/cost_table.txt @@ -106,3 +106,6 @@ EndpointProbeSwept l.put calls=6 EndpointProbeJudged l.get calls=14 EndpointProbeJudged l.list calls=1 EndpointProbeJudged l.put calls=6 +BoundedOptimize l.get calls=59 +BoundedOptimize l.list calls=4 +BoundedOptimize l.put calls=40 diff --git a/crates/omnigraph-dst/tests/scenarios.rs b/crates/omnigraph-dst/tests/scenarios.rs index 8266b52b3..9b255d6bf 100644 --- a/crates/omnigraph-dst/tests/scenarios.rs +++ b/crates/omnigraph-dst/tests/scenarios.rs @@ -2606,6 +2606,71 @@ fn merge_history_cost_rows(replay: &str) -> String { rows } +/// The bounded optimizer adds only accepted-input inventory reads; keep their +/// cost separate from the legacy direct Optimize row. +fn bounded_optimize_cost_rows(replay: &str) -> String { + struct BoundedOptimizeCost; + impl omnigraph_dst::UniverseScenario for BoundedOptimizeCost { + type Output = (); + + async fn run( + &self, + resources: &mut omnigraph_dst::memory::MemoryStorage, + _workload_seed: u64, + ) { + omnigraph_dst::cost::set_label("_bounded_optimize_setup"); + let db = Session::from_defaults( + Arc::new( + Omnigraph::init_with_storage( + &resources.root, + TEST_SCHEMA, + resources.adapter.clone(), + InitOptions::default(), + ) + .await + .unwrap(), + ), + SessionSettings::default(), + ); + db.load_jsonl(TEST_DATA, LoadMode::Overwrite).await.unwrap(); + for (name, age) in [("Eve", 34), ("Frank", 35)] { + db.mutate( + "main", + MUTATION_QUERIES, + "insert_person", + &mixed_params(&[("$name", name)], &[("$age", age)]), + ) + .await + .unwrap(); + } + let detached = db.detach(); + let db = detached.attach(Arc::new( + Omnigraph::open_with_storage(&resources.root, resources.adapter.clone()) + .await + .unwrap(), + )); + let limits = omnigraph::db::OptimizeLimits { + max_tables: 16, + max_fragments: 1024, + max_rows: 100_000, + max_source_bytes: 64 * 1024 * 1024, + max_files: 4096, + }; + omnigraph_dst::cost::set_label("BoundedOptimize"); + let report = db.optimize_with_limits_as(&limits, None).await.unwrap(); + omnigraph_dst::cost::set_label("_bounded_optimize_verify"); + assert!(report.source_rows > 0); + assert_ne!(report.before_commit, report.after_commit); + assert_eq!(person_rows_on(&db, "main").await.len(), 6); + } + } + labeled_cost_rows( + format!("shared-memory://dst-bounded-optimize-{replay}"), + &BoundedOptimizeCost, + "BoundedOptimize", + ) +} + fn labeled_cost_rows(root: String, scenario: &S, label: &str) -> String where S: omnigraph_dst::UniverseScenario, @@ -3339,6 +3404,7 @@ fn dst_bench_cost_count_golden() { table.push_str(&writer_buffer_cost_rows("a")); table.push_str(&overwrite_removal_cost_rows("a")); table.push_str(&endpoint_probe_cost_rows("a")); + table.push_str(&bounded_optimize_cost_rows("a")); let ledger2 = omnigraph_dst::cost::arm(); let _ = run_universe("shared-memory://dst-bench-cost-b", &sc); let mut table2 = ledger2.render_calls(); @@ -3350,6 +3416,7 @@ fn dst_bench_cost_count_golden() { table2.push_str(&writer_buffer_cost_rows("b")); table2.push_str(&overwrite_removal_cost_rows("b")); table2.push_str(&endpoint_probe_cost_rows("b")); + table2.push_str(&bounded_optimize_cost_rows("b")); assert_eq!( table, table2, "the counting pass must replay identically before any golden claim" diff --git a/crates/omnigraph-policy/src/lib.rs b/crates/omnigraph-policy/src/lib.rs index 3e1c18422..402f3b118 100644 --- a/crates/omnigraph-policy/src/lib.rs +++ b/crates/omnigraph-policy/src/lib.rs @@ -20,6 +20,8 @@ pub enum PolicyAction { Export, Change, SchemaApply, + /// Compact graph tables on the target branch without deleting history. + Optimize, BranchCreate, BranchDelete, BranchMerge, @@ -52,6 +54,7 @@ impl PolicyAction { Self::Export => "export", Self::Change => "change", Self::SchemaApply => "schema_apply", + Self::Optimize => "optimize", Self::BranchCreate => "branch_create", Self::BranchDelete => "branch_delete", Self::BranchMerge => "branch_merge", @@ -73,6 +76,7 @@ impl PolicyAction { self, Self::BranchCreate | Self::SchemaApply + | Self::Optimize | Self::BranchDelete | Self::BranchMerge | Self::CommitPublish @@ -93,6 +97,7 @@ impl PolicyAction { | Self::Export | Self::Change | Self::SchemaApply + | Self::Optimize | Self::BranchCreate | Self::BranchDelete | Self::BranchMerge @@ -155,6 +160,7 @@ impl FromStr for PolicyAction { "export" => Ok(Self::Export), "change" => Ok(Self::Change), "schema_apply" => Ok(Self::SchemaApply), + "optimize" => Ok(Self::Optimize), "branch_create" => Ok(Self::BranchCreate), "branch_delete" => Ok(Self::BranchDelete), "branch_merge" => Ok(Self::BranchMerge), @@ -937,6 +943,7 @@ namespace Omnigraph { action "export" appliesTo { principal: Actor, resource: Graph, context: RequestContext }; action "change" appliesTo { principal: Actor, resource: Graph, context: RequestContext }; action "schema_apply" appliesTo { principal: Actor, resource: Graph, context: RequestContext }; + action "optimize" appliesTo { principal: Actor, resource: Graph, context: RequestContext }; action "branch_create" appliesTo { principal: Actor, resource: Graph, context: RequestContext }; action "branch_delete" appliesTo { principal: Actor, resource: Graph, context: RequestContext }; action "branch_merge" appliesTo { principal: Actor, resource: Graph, context: RequestContext }; @@ -1027,7 +1034,7 @@ pub enum ResourceScope { /// modifies this branch (SchemaApply applies the new schema to it) /// or removes it (BranchDelete). Maps to /// `(branch: None, target_branch: Some(X))`. - /// Used by SchemaApply, BranchDelete. + /// Used by SchemaApply, Optimize, BranchDelete. TargetBranch(String), /// Action transitions between two branches. `source` is the /// branch being read-from / merged-from / forked-from; `target` @@ -1417,7 +1424,7 @@ rules: } #[test] - fn schema_apply_uses_target_branch_scope() { + fn schema_apply_and_optimize_use_target_branch_scope() { let policy: PolicyConfig = yaml_serde::from_str( r#" version: 1 @@ -1428,36 +1435,38 @@ rules: - id: admins-schema-apply allow: actors: { group: admins } - actions: [schema_apply] + actions: [schema_apply, optimize] target_branch_scope: protected "#, ) .unwrap(); let engine = PolicyCompiler::compile(&policy, "graph").unwrap(); - let allow = engine - .authorize( - "act-ragnor", - &PolicyRequest { - action: PolicyAction::SchemaApply, - branch: None, - target_branch: Some("main".to_string()), - }, - ) - .unwrap(); - assert!(allow.allowed); + for action in [PolicyAction::SchemaApply, PolicyAction::Optimize] { + let allow = engine + .authorize( + "act-ragnor", + &PolicyRequest { + action, + branch: None, + target_branch: Some("main".to_string()), + }, + ) + .unwrap(); + assert!(allow.allowed); - let deny = engine - .authorize( - "act-ragnor", - &PolicyRequest { - action: PolicyAction::SchemaApply, - branch: None, - target_branch: Some("feature".to_string()), - }, - ) - .unwrap(); - assert!(!deny.allowed); + let deny = engine + .authorize( + "act-ragnor", + &PolicyRequest { + action, + branch: None, + target_branch: Some("feature".to_string()), + }, + ) + .unwrap(); + assert!(!deny.allowed); + } } #[test] diff --git a/crates/omnigraph-server/src/deployment.rs b/crates/omnigraph-server/src/deployment.rs index d73138175..e5acef4c8 100644 --- a/crates/omnigraph-server/src/deployment.rs +++ b/crates/omnigraph-server/src/deployment.rs @@ -35,7 +35,7 @@ const DRAIN_TIMEOUT: Duration = Duration::from_secs(300); #[derive(Default)] pub(crate) struct DeploymentRuntime { - gate: Arc>, + pub(crate) gate: Arc>, // Process observation generation and exact current owned invocation. current: SyncMutex<(u64, Option)>, } diff --git a/crates/omnigraph-server/src/http_operations.rs b/crates/omnigraph-server/src/http_operations.rs index fe17da96b..74a4f4117 100644 --- a/crates/omnigraph-server/src/http_operations.rs +++ b/crates/omnigraph-server/src/http_operations.rs @@ -12,7 +12,7 @@ use axum::http::Method; use axum::routing::{MethodFilter, MethodRouter}; use omnigraph_api_types::operations::{HttpMethod, HttpOperation, named_operation}; -use crate::{AppState, INGEST_REQUEST_BODY_LIMIT_BYTES, deployment, handlers}; +use crate::{AppState, INGEST_REQUEST_BODY_LIMIT_BYTES, deployment, handlers, maintenance}; pub(crate) fn operation(name: &str) -> &'static HttpOperation { named_operation(name).expect("a registered HTTP operation belongs to the shared inventory") @@ -206,6 +206,13 @@ pub(crate) fn probes() -> HttpRoutes { .route("readiness", handlers::server_ready) } +pub(crate) fn maintenance() -> HttpRoutes { + HttpRoutes::new("") + .route("maintenance status", maintenance::status) + .route("optimize", maintenance::submit) + .route("maintenance lookup", maintenance::lookup) +} + #[cfg(test)] mod tests { use std::collections::BTreeSet; @@ -219,6 +226,7 @@ mod tests { let mut registered = graph().registered(); registered.extend(management().registered()); registered.extend(deployments().registered()); + registered.extend(maintenance().registered()); registered.extend(probes().registered()); registered.extend(crate::mcp::metadata_http_operations()); let count = registered.len(); diff --git a/crates/omnigraph-server/src/ingress.rs b/crates/omnigraph-server/src/ingress.rs index ffb947e8c..ce19b006d 100644 --- a/crates/omnigraph-server/src/ingress.rs +++ b/crates/omnigraph-server/src/ingress.rs @@ -149,6 +149,8 @@ pub(crate) async fn admit( .map_err(|_| ApiError::internal("Blob put limit exceeds usize"))? } else if matches!(route, "/cluster/deployments" | "/cluster/plan") { crate::deployment::REQUEST_BYTES + } else if route == "/cluster/maintenance" { + crate::maintenance::REQUEST_BYTES } else if route == "/mcp" { crate::mcp::REQUEST_BYTES } else if matches!(route, "/load/ndjson" | "/load") { @@ -339,19 +341,18 @@ mod tests { .with_operations(OperationRuntime::with_read_limit(1)); let entered = Arc::new(AtomicUsize::new(0)); let observed = Arc::clone(&entered); + let write = post(move || { + let observed = Arc::clone(&observed); + async move { + observed.fetch_add(1, Ordering::SeqCst); + StatusCode::NO_CONTENT + } + }); let app = Router::new() .route("/snapshot", get(|| async { "read response" })) .route("/query", read) - .route( - "/write", - post(move || { - let observed = Arc::clone(&observed); - async move { - observed.fetch_add(1, Ordering::SeqCst); - StatusCode::NO_CONTENT - } - }), - ) + .route("/write", write.clone()) + .route("/cluster/maintenance", write) .layer(middleware::from_fn_with_state( state.clone(), |State(state): State, request: Request, next: Next| async move { @@ -633,20 +634,34 @@ mod tests { #[tokio::test] async fn oversized_body_refuses_before_handler_execution_and_releases_capacity() { - let (app, state, entered) = router(WorkloadLimits::default()); - let response = app - .oneshot(request(Body::from(vec![ - b' '; - DEFAULT_REQUEST_BODY_LIMIT_BYTES - + 1 - ]))) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE); - assert_eq!(error_code(response).await, "payload_too_large"); - assert_eq!(entered.load(Ordering::SeqCst), 0); - assert_eq!(state.operations.snapshot().active_reads, 0); - assert_eq!(state.workload.snapshot(), WorkloadSnapshot::default()); + for (path, limit) in [ + ("/write", DEFAULT_REQUEST_BODY_LIMIT_BYTES), + ("/cluster/maintenance", crate::maintenance::REQUEST_BYTES), + ] { + let (app, state, entered) = router(WorkloadLimits::default()); + let mut request = request(Body::from(vec![b' '; limit + 1])); + *request.uri_mut() = path.parse().unwrap(); + let response = app.clone().oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE, "{path}"); + assert_eq!(error_code(response).await, "payload_too_large"); + assert_eq!(entered.load(Ordering::SeqCst), 0); + assert_eq!(state.operations.snapshot().active_reads, 0); + assert_eq!(state.workload.snapshot(), WorkloadSnapshot::default()); + // No extractor limit is installed in this fixture: the middleware + // itself must enforce the exact route ceiling before dispatch. + let response = app + .oneshot( + Request::post(path) + .body(Body::from(vec![b' '; limit])) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NO_CONTENT, "{path}"); + assert_eq!(entered.load(Ordering::SeqCst), 1); + drop(response); + assert_eq!(state.workload.snapshot(), WorkloadSnapshot::default()); + } } #[tokio::test] diff --git a/crates/omnigraph-server/src/lib.rs b/crates/omnigraph-server/src/lib.rs index a0829806b..805670f89 100644 --- a/crates/omnigraph-server/src/lib.rs +++ b/crates/omnigraph-server/src/lib.rs @@ -11,6 +11,7 @@ mod handlers; mod http_contract; mod http_operations; mod ingress; +mod maintenance; mod mcp; pub mod operations; mod redacted_cause; @@ -136,6 +137,9 @@ fn hash_bearer_token(token: &str) -> BearerTokenHash { deployment::lookup, deployment::apply, deployment::plan, + maintenance::status, + maintenance::lookup, + maintenance::submit, handlers::server_health, handlers::server_ready, handlers::server_graphs_list, @@ -421,6 +425,7 @@ pub struct AppState { // drain is not qualified native settlement, so the server never unlocks. cluster_admission: Option, deployments: Arc, + maintenance: Arc, /// Runtime routing and availability for every configured graph. /// Middleware injects an admitted `serving::GraphRequest` or refuses a blocked /// graph before collecting its request body. @@ -860,6 +865,7 @@ impl AppState { Self { cluster_admission: None, deployments: Arc::new(deployment::DeploymentRuntime::default()), + maintenance: Arc::new(maintenance::MaintenanceRuntime::default()), routing: GraphRouting { registry, config_path: None, @@ -914,6 +920,7 @@ impl AppState { Ok(Self { cluster_admission: None, deployments: Arc::new(deployment::DeploymentRuntime::default()), + maintenance: Arc::new(maintenance::MaintenanceRuntime::default()), routing: GraphRouting { registry, config_path, @@ -3710,6 +3717,16 @@ pub fn build_app(state: AppState) -> Router { routes.layer(DefaultBodyLimit::max(deployment::REQUEST_BYTES)), ) }); + let maintenance = guarded( + &state, + http_operations::maintenance().into_router(), + |routes| { + admitted( + &state, + routes.layer(DefaultBodyLimit::max(maintenance::REQUEST_BYTES)), + ) + }, + ); let catalogue_routes = Router::new() .route("/query", post(handlers::catalogue::catalogue_query)) @@ -3724,7 +3741,8 @@ pub fn build_app(state: AppState) -> Router { .nest(CATALOGUE_ROUTE_PREFIX, catalogue) .nest(handlers::CLUSTER_PATH_PREFIX, per_graph_protected) .merge(management) - .merge(deployments); + .merge(deployments) + .merge(maintenance); let mut app = http_operations::probes() .into_router() @@ -3848,6 +3866,9 @@ async fn serve_config( let tokens = token_source.load().await?; let process_defaults = ProcessDefaults::from_env() .map_err(|error| eyre!("session setting refused at startup: {error}"))?; + let maintenance_limits = maintenance::load_limits_from_env( + data_token_trust.is_some() || oidc_identity_trust.is_some(), + )?; // For runtime-state classification, "any policy configured" means // either the top-level/single-mode policy file OR a server-level @@ -3942,6 +3963,7 @@ async fn serve_config( shutdown_grace, ) .with_process_defaults(process_defaults) + .with_optional_maintenance_limits(maintenance_limits) .with_mcp_settings(mcp); deployment::initialize_boot_activation(&state); let retained_admission = state.cluster_admission.clone(); diff --git a/crates/omnigraph-server/src/maintenance.rs b/crates/omnigraph-server/src/maintenance.rs new file mode 100644 index 000000000..8aa9eec9e --- /dev/null +++ b/crates/omnigraph-server/src/maintenance.rs @@ -0,0 +1,1079 @@ +//! Server-owned optimization. Native receipts own invocation status; this +//! module only owns the task lifetime and the process admission boundary. + +use std::{ + io::Read as _, + sync::{Arc, Mutex, PoisonError}, +}; + +use axum::{Extension, Json, extract::State, http::StatusCode}; +use omnigraph::db::OptimizeLimits; +use omnigraph_cluster::{ + IdentityAuthorization, MaintenanceAcceptance, MaintenanceCompletion, MaintenanceLookup, + MaintenanceRequest, MaintenanceStatus, +}; +use serde::{Deserialize, Serialize}; +use tokio::sync::{OwnedMutexGuard, oneshot}; +use tracing::Instrument; +use utoipa::ToSchema; + +use crate::extract::{ApiJson, ApiPath, ApiQuery}; +use crate::operations::OwnedResult; +use crate::registry::RegistryCapture; +use crate::workload::IngressLease; +use crate::{ApiError, AppState, AuthenticatedActor, GraphId, GraphKey}; + +pub(crate) const REQUEST_BYTES: usize = 4096; +const CONFIG_BYTES: u64 = 4096; + +#[derive(Default)] +pub(crate) struct MaintenanceRuntime { + limits: Option, + running: Mutex<(u64, Option)>, +} + +impl MaintenanceRuntime { + async fn observe( + &self, + id: &str, + mut read: impl FnMut() -> F, + ) -> Result<(T, bool), ApiError> + where + F: std::future::Future>, + { + for _ in 0..3 { + let (generation, _) = self.observation(id); + let value = read().await?; + let (observed_generation, in_progress) = self.observation(id); + if generation == observed_generation { + return Ok((value, in_progress)); + } + } + Err(ApiError::new( + crate::api::ErrorCode::ServiceUnavailable, + "maintenance ownership changed during lookup; repeat this GET", + )) + } + + fn observation(&self, id: &str) -> (u64, bool) { + let current = self.running.lock().unwrap_or_else(PoisonError::into_inner); + (current.0, current.1.as_deref() == Some(id)) + } + + fn own(self: &Arc, id: String, gate: OwnedMutexGuard<()>) -> RunningMaintenance { + let mut current = self.running.lock().unwrap_or_else(PoisonError::into_inner); + current.0 = current.0.wrapping_add(1); + current.1 = Some(id); + RunningMaintenance { + runtime: Arc::clone(self), + gate: Some(gate), + } + } +} + +struct RunningMaintenance { + runtime: Arc, + gate: Option>, +} + +impl Drop for RunningMaintenance { + fn drop(&mut self) { + let mut current = self + .runtime + .running + .lock() + .unwrap_or_else(PoisonError::into_inner); + // A successor cannot install its marker until this transition finishes. + // Releasing the gate in a separate tuple field would let this Drop + // erase the successor's marker after it acquired that gate. + drop(self.gate.take()); + current.0 = current.0.wrapping_add(1); + current.1 = None; + } +} + +impl AppState { + /// Enable served optimization with explicit operator-owned input ceilings. + /// These are source bounds, not a process CPU/memory or storage-rate quota. + pub fn with_maintenance_limits(self, limits: OptimizeLimits) -> color_eyre::Result { + validate_limits(&limits).map_err(|error| color_eyre::eyre::eyre!(error.message))?; + Ok(self.with_optional_maintenance_limits(Some(limits))) + } + + pub(crate) fn with_optional_maintenance_limits( + mut self, + limits: Option, + ) -> Self { + self.maintenance = Arc::new(MaintenanceRuntime { + limits, + running: Mutex::new((0, None)), + }); + self + } +} + +pub(crate) fn load_limits_from_env( + has_identity_trust: bool, +) -> color_eyre::Result> { + let Some(path) = std::env::var_os("OMNIGRAPH_MAINTENANCE_CONFIG") else { + return Ok(None); + }; + load_limits_from_path(std::path::Path::new(&path), has_identity_trust).map(Some) +} + +fn load_limits_from_path( + path: &std::path::Path, + has_identity_trust: bool, +) -> color_eyre::Result { + require_capacity_profile(has_identity_trust) + .map_err(|error| color_eyre::eyre::eyre!(error.message))?; + let mut bytes = Vec::new(); + std::fs::File::open(path)? + .take(CONFIG_BYTES + 1) + .read_to_end(&mut bytes)?; + color_eyre::eyre::ensure!( + bytes.len() as u64 <= CONFIG_BYTES, + "maintenance configuration exceeds 4 KiB" + ); + let limits: OptimizeLimits = yaml_serde::from_slice(&bytes)?; + validate_limits(&limits).map_err(|error| color_eyre::eyre::eyre!(error.message))?; + Ok(limits) +} + +fn validate_limits(limits: &OptimizeLimits) -> Result<(), ApiError> { + if limits.max_tables == 0 + || limits.max_fragments == 0 + || limits.max_rows == 0 + || limits.max_files == 0 + || limits.max_source_bytes == 0 + { + return Err(ApiError::bad_request( + "optimization limits must all be positive", + )); + } + Ok(()) +} + +fn require_capacity_profile(has_identity_trust: bool) -> Result<(), ApiError> { + if has_identity_trust { + return Err(ApiError::conflict( + "served optimization with cluster-bound identity trust is disabled until resource capacity admission is available", + )); + } + Ok(()) +} + +fn ceilings(state: &AppState) -> Result<&OptimizeLimits, ApiError> { + // This is a server property, not a credential property: using a static + // token or attaching trust after limits cannot bypass required admission. + // Exact result lookup remains available without enabling fresh work. + require_capacity_profile( + state.data_token_trust.is_some() || state.oidc_identity_trust.is_some(), + )?; + state.maintenance.limits.as_ref().ok_or_else(|| ApiError::conflict( + "served optimization is disabled; the server operator must configure finite maintenance limits", + )) +} + +fn check_ceiling(request: &OptimizeLimits, ceiling: &OptimizeLimits) -> Result<(), ApiError> { + validate_limits(request)?; + if request.max_tables > ceiling.max_tables + || request.max_fragments > ceiling.max_fragments + || request.max_rows > ceiling.max_rows + || request.max_files > ceiling.max_files + || request.max_source_bytes > ceiling.max_source_bytes + { + return Err(ApiError::bad_request( + "requested optimization limits exceed server ceilings", + )); + } + Ok(()) +} + +fn identity( + actor: Option>, +) -> Result<(AuthenticatedActor, IdentityAuthorization), ApiError> { + let actor = actor + .map(|Extension(actor)| actor) + .ok_or_else(|| ApiError::forbidden("optimization requires an authenticated actor"))?; + let identity = IdentityAuthorization::authenticated(actor.actor_id_str()).map_err(refusal)?; + Ok((actor, identity)) +} + +fn owner(state: &AppState) -> Result<&omnigraph_cluster::ClusterAdmission, ApiError> { + state + .cluster_admission + .as_ref() + .ok_or_else(|| ApiError::conflict("optimization requires native serving admission")) +} + +fn refusal(error: omnigraph_cluster::Diagnostic) -> ApiError { + let message = format!("[{}] {}", error.code, error.message); + match error.code.as_str() { + "policy_denied" | "graph_policy_required" | "identity_invalid" => { + ApiError::forbidden(message) + } + "maintenance_id_invalid" | "maintenance_graph_invalid" | "maintenance_limits_invalid" => { + ApiError::bad_request(message) + } + "maintenance_write_uncertain" | "maintenance_read_error" => ApiError::internal(message), + _ => ApiError::conflict(message), + } +} + +#[derive(Deserialize, ToSchema)] +#[serde(deny_unknown_fields)] +pub(crate) struct GraphSelector { + graph: String, +} + +/// Positive source ceilings for one optimization; none may exceed the server's +/// advertised limits. They do not bound process memory, CPU or total I/O. +#[derive(Deserialize, Serialize, ToSchema)] +#[serde(deny_unknown_fields)] +#[schema(as = MaintenanceLimits)] +pub(crate) struct Limits { + #[schema(minimum = 1)] + max_tables: u32, + #[schema(minimum = 1)] + max_fragments: u64, + #[schema(minimum = 1)] + max_rows: u64, + #[schema(minimum = 1)] + max_source_bytes: u64, + #[schema(minimum = 1)] + max_files: u64, +} + +impl From for Limits { + fn from(value: OptimizeLimits) -> Self { + Self { + max_tables: value.max_tables, + max_fragments: value.max_fragments, + max_rows: value.max_rows, + max_source_bytes: value.max_source_bytes, + max_files: value.max_files, + } + } +} + +impl From for OptimizeLimits { + fn from(value: Limits) -> Self { + Self { + max_tables: value.max_tables, + max_fragments: value.max_fragments, + max_rows: value.max_rows, + max_source_bytes: value.max_source_bytes, + max_files: value.max_files, + } + } +} + +#[derive(Deserialize, Serialize, ToSchema)] +#[serde(rename_all = "snake_case")] +#[schema(as = MaintenanceKind)] +pub(crate) enum Kind { + Optimize, +} + +/// Submit exactly once. An uncertain response permits only GET observation of +/// this original operation identity, never a replacement ID or storage fallback. +#[derive(Deserialize, Serialize, ToSchema)] +#[serde(deny_unknown_fields)] +#[schema(as = MaintenanceRequest)] +pub(crate) struct SubmitRequest { + /// Canonical ledger ULID, next sequence and fresh ULID nonce, joined by `:`. + #[schema(example = "01ARZ3NDEKTSV4RRFFQ69G5FAV:1:01ARZ3NDEKTSV4RRFFQ69G5FAW")] + operation_id: String, + /// An applied graph ID. Optimization always targets its main branch. + graph: String, + kind: Kind, + limits: Limits, +} + +impl From for MaintenanceRequest { + fn from(value: SubmitRequest) -> Self { + Self { + operation_id: value.operation_id, + graph: value.graph, + kind: match value.kind { + Kind::Optimize => omnigraph_cluster::MaintenanceKind::Optimize, + }, + limits: value.limits.into(), + } + } +} + +#[derive(Serialize, ToSchema)] +#[schema(as = MaintenanceStatus)] +pub(crate) struct Status { + /// Native ledger identity used in the next operation ID. + ledger_id: String, + #[schema(minimum = 1)] + next_sequence: u64, + /// Outstanding operation for this graph, or null. Status reads create no journal. + #[schema(required = true)] + outstanding_id: Option, +} + +impl From for Status { + fn from(value: MaintenanceStatus) -> Self { + Self { + ledger_id: value.ledger_id, + next_sequence: value.next_sequence, + outstanding_id: value.outstanding_id, + } + } +} + +#[derive(Serialize, ToSchema)] +#[schema(as = MaintenanceStatusOutput)] +pub(crate) struct StatusOutput { + status: Status, + limits: Limits, +} + +// Schema-only views keep HTTP schema dependencies out of native Core and the +// engine. The native receipt remains the wire value; the parity test below +// round-trips every native variant through these views without losing fields. +#[allow(dead_code)] +mod receipt_schema { + use super::*; + + #[derive(Deserialize, Serialize, ToSchema)] + #[serde(tag = "status", rename_all = "snake_case", deny_unknown_fields)] + #[schema(as = MaintenanceLookup)] + #[allow(clippy::large_enum_variant)] + pub(super) enum Lookup { + Outstanding { + operation_id: String, + graph: String, + invocation: Invocation, + }, + Complete { + result: Result, + }, + ResultExpired { + operation_id: String, + }, + NotRecorded { + operation_id: String, + }, + IdentityMismatch { + operation_id: String, + }, + DifferentLedger { + operation_id: String, + }, + } + + #[derive(Deserialize, Serialize, ToSchema)] + #[serde(tag = "outcome", rename_all = "snake_case", deny_unknown_fields)] + #[schema(as = MaintenanceResult)] + pub(super) enum Result { + Success { + operation_id: String, + graph: String, + invocation: Invocation, + report: Report, + }, + Refused { + operation_id: String, + graph: String, + invocation: Invocation, + code: String, + message: String, + }, + Interrupted { + operation_id: String, + graph: String, + invocation: Invocation, + publication: PublicationOutcome, + }, + } + + #[derive(Deserialize, Serialize, ToSchema)] + #[serde(tag = "status", rename_all = "snake_case", deny_unknown_fields)] + #[schema(as = MaintenancePublication)] + pub(super) enum PublicationOutcome { + Unpublished, + Published { report: Report }, + } + + #[derive(Deserialize, Serialize, ToSchema)] + #[serde(deny_unknown_fields)] + #[schema(as = MaintenanceInvocation)] + pub(super) struct Invocation { + request: SubmitRequest, + actor: String, + canonical_root: String, + result_revision: u64, + /// Applied configuration contract at admission, not an execution-time head. + schema_contract: SchemaContract, + /// Accepted graph commit and source inventory, bound before physical work. + #[schema(required = true)] + input: Option, + #[schema(required = true)] + publication: Option, + #[schema(required = true)] + settlement: Option, + } + + #[derive(Deserialize, Serialize, ToSchema)] + #[serde(deny_unknown_fields)] + #[schema(as = OptimizePublication)] + struct Publication { + before_manifest_version: u64, + intent_nonce: String, + schema_contract: SchemaContract, + #[schema(required = true)] + actor: Option, + created_at: i64, + report: Report, + } + + #[derive(Deserialize, Serialize, ToSchema)] + #[serde(deny_unknown_fields)] + #[schema(as = OptimizeSettlement)] + struct Settlement { + original_digest: String, + lineage: SettlementLineage, + } + + #[derive(Deserialize, Serialize, ToSchema)] + #[serde(deny_unknown_fields)] + #[schema(as = OptimizeSettlementLineage)] + struct SettlementLineage { + graph_commit_id: String, + #[schema(required = true)] + branch: Option, + #[schema(required = true)] + actor_id: Option, + #[schema(required = true)] + merged_parent_commit_id: Option, + created_at: i64, + } + + #[derive(Deserialize, Serialize, ToSchema)] + #[serde(deny_unknown_fields)] + #[schema(as = MaintenanceSchemaContract)] + struct SchemaContract { + source_hash: String, + schema_ir_hash: String, + schema_identity_domain: String, + schema_identity_version: u32, + } + + #[derive(Deserialize, Serialize, ToSchema)] + #[serde(deny_unknown_fields)] + #[schema(as = OptimizeInput)] + struct Input { + before_commit: String, + report_max_bytes: u64, + schema_contract: SchemaContract, + source_tables: u32, + source_fragments: u64, + source_rows: u64, + source_bytes: u64, + source_files: u64, + } + + #[derive(Deserialize, Serialize, ToSchema)] + #[serde(deny_unknown_fields)] + #[schema(as = OptimizeReport)] + pub(super) struct Report { + before_commit: String, + after_commit: String, + tables: Vec, + source_tables: u32, + source_fragments: u64, + source_rows: u64, + source_bytes: u64, + source_files: u64, + } + + #[derive(Deserialize, Serialize, ToSchema)] + #[serde(deny_unknown_fields)] + #[schema(as = OptimizeTableStats)] + struct TableStats { + type_key: String, + fragments_removed: usize, + fragments_added: usize, + committed: bool, + #[schema(required = true)] + skipped: Option, + #[schema(required = true)] + published_dataset_version: Option, + #[schema(required = true)] + lance_head_version: Option, + pending_indexes: Vec, + } + + #[derive(Deserialize, Serialize, ToSchema)] + #[serde(rename_all = "snake_case")] + #[schema(as = OptimizeSkipReason)] + enum SkipReason { + DriftNeedsRepair, + } + + #[derive(Deserialize, Serialize, ToSchema)] + #[serde(deny_unknown_fields)] + #[schema(as = OptimizePendingIndex)] + struct PendingIndex { + type_key: String, + property: String, + reason: String, + } +} + +#[derive(Serialize, ToSchema)] +#[schema(as = MaintenanceOutput, example = json!({"maintenance":{"status":"not_recorded","operation_id":"01ARZ3NDEKTSV4RRFFQ69G5FAV:1:01ARZ3NDEKTSV4RRFFQ69G5FAW"},"in_progress":false}))] +pub(crate) struct Output { + /// Native lookup tagged by `status`: outstanding, complete, result_expired, + /// not_recorded, identity_mismatch or different_ledger. Complete contains + /// `result` with operation_id, graph, invocation and outcome: success with + /// its exact report, or refused with code/message. Only success proves work + /// completed; missing/expired evidence never permits replay. + #[schema(value_type = receipt_schema::Lookup)] + maintenance: MaintenanceLookup, + /// Current process owns this ID. Ownership alone is not durable acceptance; + /// an outstanding receipt without an owner remains unresolved. + in_progress: bool, +} + +#[utoipa::path( + get, path = "/cluster/maintenance", operation_id = "maintenanceStatus", tag = "cluster", + params(("graph" = String, Query, description = "Graph to optimize; scope is main")), + responses((status = 200, body = StatusOutput), (status = 400, body = crate::api::ErrorOutput), + (status = 403, body = crate::api::ErrorOutput), (status = 409, body = crate::api::ErrorOutput), + (status = 500, body = crate::api::ErrorOutput)), + security(("bearer_token" = [])) +)] +pub(crate) async fn status( + State(state): State, + actor: Option>, + ApiQuery(selector): ApiQuery, +) -> Result, ApiError> { + let (_, identity) = identity(actor)?; + let limits = ceilings(&state)?.clone(); + let status = omnigraph_cluster::maintenance_status(owner(&state)?, &selector.graph, &identity) + .await + .map_err(refusal)?; + Ok(Json(StatusOutput { + status: status.into(), + limits: limits.into(), + })) +} + +#[utoipa::path( + get, path = "/cluster/maintenance/{operation_id}", operation_id = "maintenanceLookup", tag = "cluster", + params(("operation_id" = String, Path, description = "Original maintenance identity"), + ("graph" = String, Query, description = "Exact graph of the original operation")), + responses((status = 200, body = Output), (status = 400, body = crate::api::ErrorOutput), + (status = 403, body = crate::api::ErrorOutput), (status = 409, body = crate::api::ErrorOutput), + (status = 500, body = crate::api::ErrorOutput), + (status = 503, description = "Owner changed during observation; retry this GET", body = crate::api::ErrorOutput)), + security(("bearer_token" = [])) +)] +pub(crate) async fn lookup( + State(state): State, + actor: Option>, + ApiPath(id): ApiPath, + ApiQuery(selector): ApiQuery, +) -> Result, ApiError> { + let (_, identity) = identity(actor)?; + // Receipt access remains available when new optimization is disabled. + // If the process owner finishes during the read, repeat the read so an old + // Outstanding record is never presented as ownerless terminal evidence. + let admission = owner(&state)?; + let (maintenance, in_progress) = state + .maintenance + .observe(&id, || async { + omnigraph_cluster::maintenance_lookup(admission, &selector.graph, &id, &identity) + .await + .map_err(refusal) + }) + .await?; + Ok(Json(Output { + maintenance, + in_progress, + })) +} + +#[utoipa::path( + post, path = "/cluster/maintenance", operation_id = "optimize", tag = "cluster", request_body = SubmitRequest, + responses((status = 200, body = Output), (status = 202, description = "Durable acceptance; inspect the original identity", body = Output), + (status = 400, body = crate::api::ErrorOutput), (status = 403, body = crate::api::ErrorOutput), + (status = 409, body = crate::api::ErrorOutput), (status = 415, body = crate::api::ErrorOutput), + (status = 503, body = crate::api::ErrorOutput), + (status = 500, description = "Internal failure (code `internal`) or uncertain optimization (code `outcome_unknown`); inspect the original identity without resubmission", body = crate::api::ErrorOutput)), + security(("bearer_token" = [])) +)] +pub(crate) async fn submit( + State(state): State, + actor: Option>, + Extension(ingress): Extension, + request: Result, ApiError>, +) -> Result<(StatusCode, Json), ApiError> { + let (actor, identity) = identity(actor)?; + let ApiJson(request) = request?; + let request: MaintenanceRequest = request.into(); + let admission = owner(&state)?.clone(); + let (existing, in_progress) = state + .maintenance + .observe(&request.operation_id, || async { + omnigraph_cluster::lookup_maintenance_request(&admission, &request, &identity) + .await + .map_err(refusal) + }) + .await?; + if let Some(maintenance) = existing { + return Ok(( + StatusCode::OK, + Json(Output { + maintenance, + in_progress, + }), + )); + } + check_ceiling(&request.limits, ceilings(&state)?)?; + let gate = Arc::clone(&state.deployments.gate) + .try_lock_owned() + .map_err(|_| { + ApiError::conflict( + "native configuration or maintenance is running; inspect its original identity", + ) + })?; + // Native checks current applied policy before we capture graph ownership. + omnigraph_cluster::maintenance_status(&admission, &request.graph, &identity) + .await + .map_err(refusal)?; + let key = GraphKey::cluster( + GraphId::try_from(request.graph.as_str()) + .map_err(|error| ApiError::bad_request(error.to_string()))?, + ); + let (capture, _) = state + .routing + .registry + .capture_with_policy(&state.operations, &key)?; + let RegistryCapture::Ready(graph) = capture else { + return Err(ApiError::conflict("graph is unavailable for optimization")); + }; + let reservation = state + .workload + .try_admit(&actor.actor_id, REQUEST_BYTES as u64) + .map_err(ApiError::from_workload_reject)?; + let running = state.maintenance.own(request.operation_id.clone(), gate); + let span = tracing::info_span!("maintenance.optimize", operation_id = %request.operation_id, graph = %request.graph); + let (accepted, acceptance) = oneshot::channel(); + let response = state.operations.submit( + (running, reservation, ingress, graph.clone()), + async move { + let mut uncertain = false; + let result = async { + let token = + match omnigraph_cluster::accept_maintenance(&admission, request, &identity) + .await + { + Ok(MaintenanceAcceptance::Existing(maintenance)) => { + return Ok(Output { + maintenance, + in_progress: false, + }); + } + Ok(MaintenanceAcceptance::Accepted(token)) => token, + Err(error) => { + uncertain = error.code == "maintenance_write_uncertain"; + return Err(refusal(error)); + } + }; + // Everything after acceptance retains uncertainty until the exact + // terminal record has been durably written. + uncertain = true; + tracing::info!("native optimization accepted"); + let pending = omnigraph_cluster::maintenance_lookup( + &admission, + &token.request().graph, + &token.request().operation_id, + &identity, + ) + .await + .map_err(refusal)?; + let _ = accepted.send(pending); + let limits = token.request().limits.clone(); + let token = tokio::sync::Mutex::new(token); + let binding_error = tokio::sync::Mutex::new(None); + let execution = graph + .engine + .optimize_with_limits_recorded_as( + &limits, + Some(identity.actor()), + |input| { + let token = &token; + let binding_error = &binding_error; + let admission = &admission; + async move { + let mut token = token.lock().await; + match omnigraph_cluster::bind_maintenance_input( + admission, &mut token, input, + ) + .await + { + Ok(()) => Ok(()), + Err(error) => { + let message = format!("[{}] {}", error.code, error.message); + if error.code != "maintenance_result_capacity" { + *binding_error.lock().await = Some(error); + } + Err(omnigraph::error::OmniError::manifest(message)) + } + } + } + }, + |publication| { + let token = &token; + let binding_error = &binding_error; + let admission = &admission; + async move { + let mut token = token.lock().await; + match omnigraph_cluster::bind_maintenance_publication( + admission, + &mut token, + publication, + ) + .await + { + Ok(()) => Ok(()), + Err(error) => { + *binding_error.lock().await = Some(error); + Err(omnigraph::error::OmniError::manifest( + "optimization publication could not be durably bound", + )) + } + } + } + }, + ) + .await; + // A failed journal write is not an engine preflight refusal: the + // remote outcome may be unknown even though graph effects did not + // begin. Retain the invocation and contain this process. + if let Some(error) = binding_error.into_inner() { + return Err(refusal(error)); + } + let completion = match execution { + Ok(report) => MaintenanceCompletion::Success { report }, + Err(failure) if !failure.effects_started => MaintenanceCompletion::Refused { + code: "optimization_preflight_refused".into(), + message: failure.error.to_string().chars().take(256).collect(), + }, + Err(failure) => return Err(ApiError::from_omni(failure.error)), + }; + let result = omnigraph_cluster::complete_maintenance( + &admission, + token.into_inner(), + completion, + ) + .await + .map_err(refusal)?; + uncertain = false; + tracing::info!("native optimization result retained"); + Ok(Output { + maintenance: MaintenanceLookup::Complete { result }, + in_progress: false, + }) + } + .await; + if uncertain { + tracing::error!( + "native optimization unresolved; retain original identity and ownership" + ); + } + OwnedResult { result, uncertain } + } + .instrument(span), + )?; + let result = response.result(); + tokio::pin!(result); + tokio::select! { + biased; + result = &mut result => result.map(|output| (StatusCode::OK, Json(output))), + accepted = acceptance => match accepted { + Ok(maintenance) => Ok((StatusCode::ACCEPTED, Json(Output { maintenance, in_progress: true }))), + Err(_) => result.await.map(|output| (StatusCode::OK, Json(output))), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn startup_identity_trust_cannot_enable_source_only_maintenance_limits() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("maintenance.yaml"); + std::fs::write( + &path, + "max_tables: 10\nmax_fragments: 100\nmax_rows: 1000\nmax_source_bytes: 100000\nmax_files: 100\n", + ) + .unwrap(); + assert!(load_limits_from_path(&path, false).is_ok()); + let error = load_limits_from_path(&path, true).unwrap_err(); + assert!(error.to_string().contains("resource capacity admission")); + } + + #[test] + fn documented_receipt_preserves_every_native_variant_and_field() { + let id = "01ARZ3NDEKTSV4RRFFQ69G5FAV:1:01ARZ3NDEKTSV4RRFFQ69G5FAW"; + let mut invocation = omnigraph_cluster::MaintenanceInvocation { + request: MaintenanceRequest { + operation_id: id.into(), + graph: "knowledge".into(), + kind: omnigraph_cluster::MaintenanceKind::Optimize, + limits: OptimizeLimits { + max_tables: 32, + max_fragments: 1000, + max_rows: 100000, + max_source_bytes: 134217728, + max_files: 2000, + }, + }, + actor: "act-operator".into(), + canonical_root: "file:///graph-root".into(), + result_revision: 2, + schema_contract: omnigraph::db::SchemaContractDigest { + source_hash: "source".into(), + schema_ir_hash: "ir".into(), + schema_identity_domain: "domain".into(), + schema_identity_version: 1, + }, + input: None, + publication: None, + settlement: None, + }; + let mut lookups = vec![MaintenanceLookup::Outstanding { + operation_id: id.into(), + graph: "knowledge".into(), + invocation: invocation.clone(), + }]; + let input = omnigraph::db::OptimizeInput { + before_commit: "before".into(), + report_max_bytes: 16384, + schema_contract: invocation.schema_contract.clone(), + source_tables: 2, + source_fragments: 10, + source_rows: 100, + source_bytes: 2000, + source_files: 20, + }; + invocation.input = Some(input.clone()); + lookups.push(MaintenanceLookup::Outstanding { + operation_id: id.into(), + graph: "knowledge".into(), + invocation: invocation.clone(), + }); + let tables = serde_json::from_value(serde_json::json!([ + {"type_key":"node:Person","fragments_removed":4,"fragments_added":1, + "committed":true,"skipped":null,"published_dataset_version":null, + "lance_head_version":null,"pending_indexes":[{"type_key":"node:Person", + "property":"embedding","reason":"no trainable vectors"}]}, + {"type_key":"edge:Knows","fragments_removed":0,"fragments_added":0, + "committed":false,"skipped":"drift_needs_repair","published_dataset_version":4, + "lance_head_version":5,"pending_indexes":[]} + ])) + .unwrap(); + let report = omnigraph::db::OptimizeReport { + before_commit: input.before_commit.clone(), + after_commit: "after".into(), + tables, + source_tables: input.source_tables, + source_fragments: input.source_fragments, + source_rows: input.source_rows, + source_bytes: input.source_bytes, + source_files: input.source_files, + }; + invocation.publication = Some(omnigraph::db::OptimizePublication { + before_manifest_version: 3, + intent_nonce: "01ARZ3NDEKTSV4RRFFQ69G5FAW".into(), + schema_contract: invocation.schema_contract.clone(), + actor: Some(invocation.actor.clone()), + created_at: 42, + report: report.clone(), + }); + invocation.settlement = Some(serde_json::from_value(serde_json::json!({ + "original_digest":"a".repeat(64), + "lineage":{"graph_commit_id":"01ARZ3NDEKTSV4RRFFQ69G5FAX", + "branch":null,"actor_id":"act-recovery","merged_parent_commit_id":null,"created_at":43} + })).unwrap()); + lookups.push(MaintenanceLookup::Outstanding { + operation_id: id.into(), + graph: "knowledge".into(), + invocation: invocation.clone(), + }); + for completion in [ + MaintenanceCompletion::Success { + report: report.clone(), + }, + MaintenanceCompletion::Refused { + code: "optimization_preflight_refused".into(), + message: "source exceeds configured ceiling".into(), + }, + MaintenanceCompletion::Interrupted { + publication: omnigraph_cluster::MaintenancePublication::Unpublished, + }, + MaintenanceCompletion::Interrupted { + publication: omnigraph_cluster::MaintenancePublication::Published { report }, + }, + ] { + lookups.push(MaintenanceLookup::Complete { + result: omnigraph_cluster::MaintenanceResult { + operation_id: id.into(), + graph: "knowledge".into(), + invocation: invocation.clone(), + completion, + }, + }); + } + lookups.extend([ + MaintenanceLookup::ResultExpired { + operation_id: id.into(), + }, + MaintenanceLookup::NotRecorded { + operation_id: id.into(), + }, + MaintenanceLookup::IdentityMismatch { + operation_id: id.into(), + }, + MaintenanceLookup::DifferentLedger { + operation_id: id.into(), + }, + ]); + for lookup in lookups { + let native = serde_json::to_value(lookup).unwrap(); + let documented: receipt_schema::Lookup = + serde_json::from_value(native.clone()).unwrap(); + assert_eq!(serde_json::to_value(documented).unwrap(), native); + } + } + + #[test] + fn documented_request_preserves_native_shape_and_refuses_unknown_input() { + let body = serde_json::json!({ + "operation_id":"01ARZ3NDEKTSV4RRFFQ69G5FAV:1:01ARZ3NDEKTSV4RRFFQ69G5FAW", + "graph":"knowledge", "kind":"optimize", + "limits":{"max_tables":32,"max_fragments":1000,"max_rows":100000,"max_source_bytes":134217728,"max_files":2000} + }); + let native: MaintenanceRequest = serde_json::from_value::(body.clone()) + .unwrap() + .into(); + assert_eq!(serde_json::to_value(native).unwrap(), body); + for malformed in [ + { + let mut value = body.clone(); + value["actor"] = "caller-selected".into(); + value + }, + { + let mut value = body.clone(); + value["kind"] = "cleanup".into(); + value + }, + { + let mut value = body.clone(); + value["limits"]["unbounded"] = true.into(); + value + }, + { + let mut value = body; + value["limits"].as_object_mut().unwrap().remove("max_files"); + value + }, + ] { + assert!(serde_json::from_value::(malformed).is_err()); + } + } + + #[tokio::test] + async fn owner_turnover_keeps_the_gate_until_observation_is_cleared() { + struct Dropped(std::sync::mpsc::Sender<()>); + impl Drop for Dropped { + fn drop(&mut self) { + self.0.send(()).unwrap(); + } + } + + let runtime = Arc::new(MaintenanceRuntime::default()); + let gate = Arc::new(tokio::sync::Mutex::new(())); + let permit = Arc::clone(&gate).lock_owned().await; + let running = runtime.own("old".into(), permit); + let (sent, received) = std::sync::mpsc::channel(); + // Hold observation so teardown cannot clear the old marker. A drop + // signal starts teardown deterministically; the owner must still hold + // the gate while waiting to clear its marker. + let early_successor = { + let current = runtime.running.lock().unwrap(); + let reservation = (Dropped(sent), running); + let dropping = std::thread::spawn(move || drop(reservation)); + received + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + let early_successor = Arc::clone(&gate).try_lock_owned().ok(); + drop(current); + dropping.join().unwrap(); + early_successor + }; + assert!( + early_successor.is_none(), + "a successor could acquire the gate before the old owner cleared its marker" + ); + assert!(!runtime.observation("old").1); + let permit = Arc::clone(&gate).lock_owned().await; + let successor = runtime.own("new".into(), permit); + assert!(runtime.observation("new").1); + drop(successor); + assert!(!runtime.observation("new").1); + } + + #[tokio::test] + async fn observation_rechecks_complete_owner_turnover_and_bounds_churn() { + let runtime = Arc::new(MaintenanceRuntime::default()); + let gate = Arc::new(tokio::sync::Mutex::new(())); + let mut reads = 0; + let (value, running) = runtime + .observe("original", || { + reads += 1; + if reads == 1 { + // Both observations would say false without the generation: + // an entire accepted operation ran during the first read. + let owner = runtime.own( + "original".into(), + Arc::clone(&gate).try_lock_owned().unwrap(), + ); + drop(owner); + } + std::future::ready(Ok(reads)) + }) + .await + .unwrap(); + assert_eq!(value, 2); + assert!(!running); + + let mut reads = 0; + let result = runtime + .observe("original", || { + reads += 1; + drop(runtime.own( + "original".into(), + Arc::clone(&gate).try_lock_owned().unwrap(), + )); + std::future::ready(Ok(())) + }) + .await; + assert!( + result.is_err(), + "owner churn must never make stale evidence terminal" + ); + assert_eq!(reads, 3, "one request cannot observe indefinitely"); + } +} diff --git a/crates/omnigraph-server/tests/boot_settings.rs b/crates/omnigraph-server/tests/boot_settings.rs index 2f7d97190..b2d7cdefe 100644 --- a/crates/omnigraph-server/tests/boot_settings.rs +++ b/crates/omnigraph-server/tests/boot_settings.rs @@ -79,6 +79,9 @@ async fn deployment_routes_refuse_with_the_error_body() { (Method::GET, "/cluster/deployments/some-id"), (Method::POST, "/cluster/plan"), (Method::POST, "/cluster/deployments"), + (Method::GET, "/cluster/maintenance?graph=knowledge"), + (Method::GET, "/cluster/maintenance/some-id?graph=knowledge"), + (Method::POST, "/cluster/maintenance"), ] { let (status, body) = json_response( &open, @@ -1539,7 +1542,9 @@ mod owned_shutdown { .env(MODE, mode) .env( "OMNIGRAPH_SERVER_BEARER_TOKENS_JSON", - if mode.starts_with("startup-") { + if mode.starts_with("maintenance-") { + r#"{"operator":"operator-token","reader":"reader-token"}"# + } else if mode.starts_with("startup-") { r#"{"startup-operator":"startup-secret"}"# } else if mode == "v2-finish" { r#"{"restart-operator":"restart-secret"}"# @@ -1550,6 +1555,13 @@ mod owned_shutdown { .env_remove("OMNIGRAPH_SERVER_BEARER_TOKEN") .env_remove("OMNIGRAPH_SERVER_BEARER_TOKENS_FILE") .env_remove("OMNIGRAPH_SERVER_BEARER_TOKENS_AWS_SECRET") + .env_remove("OMNIGRAPH_MAINTENANCE_CONFIG") + .envs(mode.starts_with("maintenance-").then(|| { + ( + "OMNIGRAPH_MAINTENANCE_CONFIG", + root.join("maintenance.yaml"), + ) + })) .env("OMNIGRAPH_PER_ACTOR_INFLIGHT_MAX", "1") .stdout(Stdio::piped()) .stderr(Stdio::inherit()) @@ -1675,6 +1687,49 @@ mod owned_shutdown { return; }; let mode = std::env::var(MODE).unwrap(); + if mode.starts_with("maintenance-") { + let mut config = omnigraph_server::load_server_settings( + Some(&root), + Some("127.0.0.1:0".into()), + false, + true, + ) + .await + .unwrap(); + config.shutdown_grace = if mode == "maintenance-cutoff" { + Duration::from_millis(500) + } else { + Duration::from_secs(10) + }; + let seam = if mode == "maintenance-after-publication" { + &omnigraph::seams::catalog::GRAPH_PUBLISH_AFTER_MANIFEST_COMMIT + } else { + &omnigraph::seams::catalog::OPTIMIZE_POST_PHASE_B_PRE_MANIFEST_COMMIT + }; + let (_guard, hold) = seam.hold(); + let control_root = root.clone(); + std::thread::spawn(move || { + hold.wait_until_reached(); + fs::write(control_root.join("maintenance-held"), b"held").unwrap(); + let deadline = Instant::now() + Duration::from_secs(20); + while !control_root.join("release").exists() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + hold.release(); + }); + let signal_root = root.clone(); + let mut terminate = + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()).unwrap(); + tokio::spawn(async move { + terminate.recv().await.unwrap(); + fs::write(signal_root.join("signal-observed"), b"observed").unwrap(); + }); + let result = omnigraph_server::serve(config).await; + if let Err(error) = &result { + eprintln!("maintenance shutdown refused: {error:?}"); + } + std::process::exit(if result.is_ok() { 0 } else { 2 }); + } if mode.starts_with("startup-") { let mut config = server_config( &root, @@ -2328,6 +2383,337 @@ mod owned_shutdown { ); } + async fn optimization_disconnect_and_shutdown(mode: &str) { + let temp = converged_cluster_dir("").await; + let root = temp.path(); + fs::write(root.join("maintenance-policy.yaml"), "version: 1\ngroups:\n operators: [operator]\n readers: [reader]\nrules:\n - id: optimize\n allow:\n actors: {group: operators}\n actions: [optimize, read, change]\n - id: read\n allow:\n actors: {group: readers}\n actions: [read]\n").unwrap(); + let mut config = fs::read_to_string(root.join("cluster.yaml")).unwrap(); + config.push_str("\npolicies:\n maintenance:\n file: ./maintenance-policy.yaml\n applies_to: [knowledge]\n"); + fs::write(root.join("cluster.yaml"), config).unwrap(); + apply_cluster_fixture(root).await; + fs::write(root.join("maintenance.yaml"), "max_tables: 100\nmax_fragments: 1000\nmax_rows: 100000\nmax_source_bytes: 134217728\nmax_files: 10000\n").unwrap(); + let graph = root.join("graphs/knowledge.omni"); + let before = { + let db = session(Omnigraph::open(graph.to_str().unwrap()).await.unwrap()); + for name in ["Ada", "Grace", "Barbara", "Edsger"] { + db.load_jsonl( + &serde_json::json!({"type":"Person","data":{"name":name}}).to_string(), + omnigraph::loader::LoadMode::Append, + ) + .await + .unwrap(); + } + db.list_commits(None).await.unwrap()[0] + .graph_commit_id + .clone() + }; + let authority = fs::read(root.join("__cluster/state.json")).unwrap(); + let (mut child, address, output_thread) = spawn_owned_child(root, mode); + wait_ready(&address, &mut child.0, "serving").await; + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(5)) + .build() + .unwrap(); + let get = |path: String| { + client + .get(format!("http://{address}{path}")) + .header(HTTP_API_CONTRACT_HEADER, HTTP_API_CONTRACT) + .bearer_auth("operator-token") + }; + let status: Value = get("/cluster/maintenance?graph=knowledge".into()) + .send() + .await + .unwrap() + .error_for_status() + .unwrap() + .json() + .await + .unwrap(); + let native_status: omnigraph_cluster::MaintenanceStatus = + serde_json::from_value(status["status"].clone()).unwrap(); + let id = native_status.next_operation_id(); + let body = serde_json::json!({"operation_id":id,"graph":"knowledge","kind":"optimize","limits":status["limits"]}); + let body_text = body.to_string(); + let mut socket = TcpStream::connect(&address).unwrap(); + socket + .set_write_timeout(Some(Duration::from_secs(3))) + .unwrap(); + write!(socket, "POST /cluster/maintenance HTTP/1.1\r\nHost: {address}\r\nConnection: close\r\nAuthorization: Bearer operator-token\r\nContent-Type: application/json\r\n{HTTP_API_CONTRACT_HEADER}: {HTTP_API_CONTRACT}\r\nContent-Length: {}\r\n\r\n{body_text}", body_text.len()).unwrap(); + socket.flush().unwrap(); + wait_marker(&root.join("maintenance-held"), &mut child.0); + // Drop the transport without reading its acceptance response. + drop(socket); + let path = format!("/cluster/maintenance/{id}?graph=knowledge"); + let pending: Value = get(path.clone()) + .send() + .await + .unwrap() + .error_for_status() + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(pending["maintenance"]["status"], "outstanding", "{pending}"); + assert_eq!(pending["in_progress"], true); + let duplicate: Value = client + .post(format!("http://{address}/cluster/maintenance")) + .header(HTTP_API_CONTRACT_HEADER, HTTP_API_CONTRACT) + .bearer_auth("operator-token") + .json(&body) + .send() + .await + .unwrap() + .error_for_status() + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(duplicate, pending, "duplicate delivery is observation only"); + if mode != "maintenance-after-publication" { + // Reads proceed while detached effects await publication. The other case + // freezes the publication's coordinator critical section, where + // a newly captured snapshot may wait for atomic publication. + let read: Value = client.post(format!("http://{address}/graphs/knowledge/query")) + .header(HTTP_API_CONTRACT_HEADER, HTTP_API_CONTRACT).bearer_auth("reader-token") + .json(&serde_json::json!({"query":"query names() { match { $p: Person } return { $p.name } }","name":"names"})) + .send().await.unwrap().error_for_status().unwrap().json().await.unwrap(); + for name in ["Ada", "Grace", "Barbara", "Edsger"] { + assert!( + read.to_string().contains(name), + "read blocked or changed while optimizing: {read}" + ); + } + } + let lock = fs::read(root.join("__cluster/lock.json")).unwrap(); + let signal = if mode == "maintenance-after-publication" { + libc::SIGKILL + } else { + libc::SIGTERM + }; + assert_eq!( + unsafe { libc::kill(child.0.id() as libc::pid_t, signal) }, + 0 + ); + if signal == libc::SIGTERM { + wait_marker(&root.join("signal-observed"), &mut child.0); + assert!( + child.0.try_wait().unwrap().is_none(), + "shutdown lost its accepted owner" + ); + assert_eq!(fs::read(root.join("__cluster/lock.json")).unwrap(), lock); + if mode == "maintenance-finish" { + fs::write(root.join("release"), b"release").unwrap(); + } + } + let deadline = Instant::now() + Duration::from_secs(15); + let exit = loop { + if let Some(exit) = child.0.try_wait().unwrap() { + break exit; + } + assert!( + Instant::now() < deadline, + "optimization shutdown exceeded its bound" + ); + tokio::time::sleep(Duration::from_millis(10)).await; + }; + output_thread.join().unwrap(); + let journal: Value = + serde_json::from_slice(&fs::read(root.join("__cluster/maintenance.json")).unwrap()) + .unwrap(); + assert_eq!( + fs::read(root.join("__cluster/state.json")).unwrap(), + authority + ); + let db = Omnigraph::open_read_only(graph.to_str().unwrap()) + .await + .unwrap(); + let after = db.list_commits(None).await.unwrap()[0] + .graph_commit_id + .clone(); + if mode == "maintenance-finish" { + assert_eq!(exit.code(), Some(0)); + assert!(!root.join("__cluster/lock.json").exists()); + assert!(journal["outstanding"].is_null()); + assert_ne!(before, after, "fixture must physically compact and publish"); + let (mut successor, next_address, next_output) = + spawn_owned_child(root, "maintenance-finish"); + wait_ready(&next_address, &mut successor.0, "serving").await; + let result: Value = client + .get(format!("http://{next_address}{path}")) + .header(HTTP_API_CONTRACT_HEADER, HTTP_API_CONTRACT) + .bearer_auth("operator-token") + .send() + .await + .unwrap() + .error_for_status() + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(result["in_progress"], false); + let result = &result["maintenance"]["result"]; + assert_eq!(result["operation_id"], id); + assert_eq!(result["outcome"], "success"); + assert_eq!(result["report"]["before_commit"], before); + assert_eq!(result["report"]["after_commit"], after); + assert_eq!( + unsafe { libc::kill(successor.0.id() as libc::pid_t, libc::SIGTERM) }, + 0 + ); + let deadline = Instant::now() + Duration::from_secs(15); + loop { + if let Some(exit) = successor.0.try_wait().unwrap() { + assert_eq!(exit.code(), Some(0)); + break; + } + assert!( + Instant::now() < deadline, + "successor shutdown exceeded bound" + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } + next_output.join().unwrap(); + } else { + assert!(!exit.success()); + assert_eq!(fs::read(root.join("__cluster/lock.json")).unwrap(), lock); + assert_eq!(journal["outstanding"]["request"]["operation_id"], id); + if mode == "maintenance-cutoff" { + assert_eq!(before, after); + } else { + assert_ne!(before, after); + } + assert!( + cluster_settings(root).await.is_err(), + "uncertain writer cannot restart" + ); + // The predecessor is reaped and this fixture uses local storage. + // Recover only its original ID under a fresh exact native lock. + let lock_id = omnigraph_cluster::deployment_status( + root.to_str().unwrap(), + None, + &omnigraph_cluster::DeploymentCaller::storage_owner(None), + ) + .await + .unwrap() + .lock_id + .unwrap(); + omnigraph_cluster::force_unlock_storage_root(root.to_str().unwrap(), &lock_id) + .await + .unwrap(); + let recovery = omnigraph_cluster::acquire_cluster_admission( + root.to_str().unwrap(), + omnigraph_cluster::ClusterAdmissionPurpose::MaintenanceRecovery { + operation_id: id.clone(), + }, + ) + .await + .unwrap() + .unwrap(); + let identity = + omnigraph_cluster::IdentityAuthorization::authenticated("operator").unwrap(); + let recovered = + omnigraph_cluster::reconcile_maintenance(&recovery, "knowledge", &id, &identity) + .await + .unwrap(); + assert_eq!(recovered.operation_id, id); + match &recovered.completion { + omnigraph_cluster::MaintenanceCompletion::Interrupted { + publication: omnigraph_cluster::MaintenancePublication::Published { report }, + } => { + assert_eq!(mode, "maintenance-after-publication"); + assert_eq!(report.before_commit, before); + assert_eq!(report.after_commit, after); + assert!(!report.tables.is_empty()); + } + omnigraph_cluster::MaintenanceCompletion::Interrupted { + publication: omnigraph_cluster::MaintenancePublication::Unpublished, + } => assert_eq!(mode, "maintenance-cutoff"), + other => panic!("recovery must not infer full success: {other:?}"), + } + let recovered_bytes = fs::read(root.join("__cluster/maintenance.json")).unwrap(); + assert_eq!( + omnigraph_cluster::reconcile_maintenance(&recovery, "knowledge", &id, &identity) + .await + .unwrap(), + recovered + ); + assert_eq!( + fs::read(root.join("__cluster/maintenance.json")).unwrap(), + recovered_bytes + ); + recovery.release_after_settlement().await.unwrap(); + let reopened = Omnigraph::open_read_only(graph.to_str().unwrap()) + .await + .unwrap(); + assert_eq!( + reopened.list_commits(None).await.unwrap()[0].graph_commit_id, + after, + "recovery must not replay optimization" + ); + let (mut successor, next_address, next_output) = + spawn_owned_child(root, "maintenance-finish"); + wait_ready(&next_address, &mut successor.0, "serving").await; + let response: Value = client + .get(format!("http://{next_address}{path}")) + .header(HTTP_API_CONTRACT_HEADER, HTTP_API_CONTRACT) + .bearer_auth("operator-token") + .send() + .await + .unwrap() + .error_for_status() + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(response["maintenance"]["status"], "complete"); + assert_eq!( + response["maintenance"]["result"], + serde_json::to_value(&recovered).unwrap() + ); + assert_eq!(response["in_progress"], false); + let read: Value = client.post(format!("http://{next_address}/graphs/knowledge/query")) + .header(HTTP_API_CONTRACT_HEADER, HTTP_API_CONTRACT).bearer_auth("reader-token") + .json(&serde_json::json!({"query":"query names() { match { $p: Person } return { $p.name } }","name":"names"})) + .send().await.unwrap().error_for_status().unwrap().json().await.unwrap(); + for name in ["Ada", "Grace", "Barbara", "Edsger"] { + assert!(read.to_string().contains(name)); + } + assert_eq!( + unsafe { libc::kill(successor.0.id() as libc::pid_t, libc::SIGTERM) }, + 0 + ); + let deadline = Instant::now() + Duration::from_secs(15); + loop { + if let Some(exit) = successor.0.try_wait().unwrap() { + assert_eq!(exit.code(), Some(0)); + break; + } + assert!( + Instant::now() < deadline, + "recovered successor shutdown exceeded bound" + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } + next_output.join().unwrap(); + assert!(!root.join("__cluster/lock.json").exists()); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn served_optimization_disconnect_settles_before_clean_restart() { + optimization_disconnect_and_shutdown("maintenance-finish").await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn served_optimization_shutdown_cutoff_retains_original_authority() { + optimization_disconnect_and_shutdown("maintenance-cutoff").await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn served_optimization_crash_after_publication_retains_unknown_receipt() { + optimization_disconnect_and_shutdown("maintenance-after-publication").await; + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn disconnected_write_and_shutdown_share_ownership_finish() { disconnected_write_and_shutdown_share_ownership("finish").await; @@ -2554,6 +2940,385 @@ async fn live_deployment_installs_updates_preserves_and_removes_catalogue_policy live_deployment_fixture("catalogue").await; } +/// Optimization needs the real admitted owner, installed policy, owned HTTP +/// task and durable receipt. Engine-only tests cannot establish this boundary. +#[tokio::test(flavor = "multi_thread")] +async fn served_optimization_enforces_bounds_and_retains_exact_results() { + live_deployment_fixture("maintenance").await; +} + +async fn served_optimization_journey( + root: &std::path::Path, + state: &AppState, + engine: &std::sync::Arc, +) { + use omnigraph::db::OptimizeLimits; + use omnigraph_cluster::{MaintenanceKind, MaintenanceRequest, MaintenanceStatus}; + use std::time::Duration; + + fn submit(request: &MaintenanceRequest, token: &str) -> Request { + Request::post("/cluster/maintenance") + .header(HTTP_API_CONTRACT_HEADER, HTTP_API_CONTRACT) + .header("authorization", format!("Bearer {token}")) + .header("content-type", "application/json") + .body(Body::from(serde_json::to_vec(request).unwrap())) + .unwrap() + } + async fn finish(app: &Router, id: &str) -> Value { + tokio::time::timeout(Duration::from_secs(30), async { + loop { + let (code, body) = json_response( + app, + get_request( + &format!("/cluster/maintenance/{id}?graph=knowledge"), + "operator-token", + ), + ) + .await; + assert_eq!(code, StatusCode::OK, "{body}"); + if body["in_progress"] == false { + assert_eq!(body["maintenance"]["status"], "complete", "{body}"); + return body["maintenance"]["result"].clone(); + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("optimization must retain its owner after the HTTP response") + } + let limits = OptimizeLimits { + max_tables: 100, + max_fragments: 1000, + max_rows: 100_000, + max_source_bytes: 128 * 1024 * 1024, + max_files: 10_000, + }; + let disabled = build_app(state.clone()); + let (code, _) = json_response( + &disabled, + get_request("/cluster/maintenance?graph=knowledge", "operator-token"), + ) + .await; + assert_eq!(code, StatusCode::CONFLICT); + assert!(!root.join("__cluster/maintenance.json").exists()); + let app = build_app( + state + .clone() + .with_maintenance_limits(limits.clone()) + .unwrap(), + ); + let (code, _) = json_response( + &app, + get_request("/cluster/maintenance?graph=knowledge", "reader-token"), + ) + .await; + assert_eq!(code, StatusCode::FORBIDDEN); + let (code, body) = json_response( + &app, + get_request("/cluster/maintenance?graph=knowledge", "operator-token"), + ) + .await; + assert_eq!(code, StatusCode::OK, "{body}"); + let status: MaintenanceStatus = serde_json::from_value(body["status"].clone()).unwrap(); + let mut request = MaintenanceRequest { + operation_id: status.next_operation_id(), + graph: "knowledge".into(), + kind: MaintenanceKind::Optimize, + limits: limits.clone(), + }; + request.limits.max_tables += 1; + let (code, _) = json_response(&app, submit(&request, "operator-token")).await; + assert_eq!(code, StatusCode::BAD_REQUEST); + assert!( + !root.join("__cluster/maintenance.json").exists(), + "over-ceiling request must not reserve an identity" + ); + request.limits = limits.clone(); + let (code, _) = json_response(&app, submit(&request, "reader-token")).await; + assert_eq!(code, StatusCode::FORBIDDEN); + assert!(!root.join("__cluster/maintenance.json").exists()); + + served_optimization_identity_profiles_refuse_new_work(root, state, engine, &request, None) + .await; + + let session = + omnigraph::Session::from_defaults(std::sync::Arc::clone(engine), Default::default()); + for name in ["Ada", "Grace", "Edsger", "Barbara"] { + session + .mutate_as( + "main", + &format!("query add() {{ insert Person {{ name: \"{name}\" }} }}"), + "add", + &Default::default(), + Some("operator"), + ) + .await + .unwrap(); + } + let before = engine.list_commits(None).await.unwrap()[0] + .graph_commit_id + .clone(); + let authority = fs::read(root.join("__cluster/state.json")).unwrap(); + + // Preflight refuses the complete accepted source before any graph effect, + // and its exact refusal consumes an ID without blocking later valid work. + request.limits.max_rows = 1; + let (code, body) = json_response(&app, submit(&request, "operator-token")).await; + assert!( + matches!(code, StatusCode::OK | StatusCode::ACCEPTED), + "{body}" + ); + let refused = finish(&app, &request.operation_id).await; + assert_eq!(refused["outcome"], "refused", "{refused}"); + assert_eq!( + engine.list_commits(None).await.unwrap()[0].graph_commit_id, + before + ); + + let (_, body) = json_response( + &app, + get_request("/cluster/maintenance?graph=knowledge", "operator-token"), + ) + .await; + let status: MaintenanceStatus = serde_json::from_value(body["status"].clone()).unwrap(); + request.operation_id = status.next_operation_id(); + request.limits = limits; + let (code, body) = json_response(&app, submit(&request, "operator-token")).await; + assert!( + matches!(code, StatusCode::OK | StatusCode::ACCEPTED), + "{body}" + ); + // The request waiter is gone here; the server owns completion independently. + let receipt = finish(&app, &request.operation_id).await; + assert_eq!(receipt["outcome"], "success", "{receipt}"); + assert_eq!(receipt["invocation"]["actor"], "operator"); + assert_eq!( + receipt["invocation"]["input"]["before_commit"], + before.to_string() + ); + assert_eq!(receipt["report"]["before_commit"], before.to_string()); + let after = engine.list_commits(None).await.unwrap()[0] + .graph_commit_id + .clone(); + assert_eq!(receipt["report"]["after_commit"], after.to_string()); + assert_eq!( + fs::read(root.join("__cluster/state.json")).unwrap(), + authority, + "maintenance never rewrites deployment authority or results" + ); + + let (code, duplicate) = json_response(&app, submit(&request, "operator-token")).await; + assert_eq!(code, StatusCode::OK, "{duplicate}"); + assert_eq!(duplicate["maintenance"]["result"], receipt); + assert_eq!( + engine.list_commits(None).await.unwrap()[0].graph_commit_id, + after + ); + let (_, next) = json_response( + &app, + get_request("/cluster/maintenance?graph=knowledge", "operator-token"), + ) + .await; + let next: MaintenanceStatus = serde_json::from_value(next["status"].clone()).unwrap(); + let mut fresh = request.clone(); + fresh.operation_id = next.next_operation_id(); + served_optimization_identity_profiles_refuse_new_work( + root, + state, + engine, + &fresh, + Some((&request, &receipt)), + ) + .await; + request.limits.max_rows -= 1; + let (code, _) = json_response(&app, submit(&request, "operator-token")).await; + assert_eq!( + code, + StatusCode::CONFLICT, + "same identity cannot accept different input" + ); + let (code, retained) = json_response( + &disabled, + get_request( + &format!( + "/cluster/maintenance/{}?graph=knowledge", + request.operation_id + ), + "operator-token", + ), + ) + .await; + assert_eq!(code, StatusCode::OK, "{retained}"); + assert_eq!( + retained["maintenance"]["result"], receipt, + "disabling new work never hides retained receipts" + ); + + // Once detached physical work has started, an error must retain the + // invocation and close process admission, even if no graph commit appeared. + for name in ["Donald", "Margaret", "Alan", "Frances"] { + session + .mutate_as( + "main", + &format!("query add() {{ insert Person {{ name: \"{name}\" }} }}"), + "add", + &Default::default(), + Some("operator"), + ) + .await + .unwrap(); + } + let before_failure = engine.list_commits(None).await.unwrap()[0] + .graph_commit_id + .clone(); + let (_, body) = json_response( + &app, + get_request("/cluster/maintenance?graph=knowledge", "operator-token"), + ) + .await; + let status: MaintenanceStatus = serde_json::from_value(body["status"].clone()).unwrap(); + request.operation_id = status.next_operation_id(); + let _fault = omnigraph::seams::catalog::OPTIMIZE_POST_TABLE_EFFECT.fire_once_at(1); + let (code, body) = json_response(&app, submit(&request, "operator-token")).await; + assert!( + matches!( + code, + StatusCode::ACCEPTED | StatusCode::INTERNAL_SERVER_ERROR + ), + "{body}" + ); + tokio::time::timeout(Duration::from_secs(30), async { + while state.operation_runtime().snapshot().uncertain_writes == 0 { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("physical-effect failure must be contained"); + assert!(state.operation_runtime().snapshot().closed); + let journal: Value = + serde_json::from_slice(&fs::read(root.join("__cluster/maintenance.json")).unwrap()) + .unwrap(); + assert_eq!( + journal["outstanding"]["request"]["operation_id"], + request.operation_id + ); + assert_eq!( + journal["outstanding"]["input"]["before_commit"], + before_failure.to_string() + ); + assert_eq!( + engine.list_commits(None).await.unwrap()[0].graph_commit_id, + before_failure + ); +} + +/// The CLI's target-kind refusal is not a capacity boundary. Direct HTTP must +/// refuse on either identity profile, even for a valid static credential and +/// regardless of embedding builder order, while retaining exact receipt reads. +async fn served_optimization_identity_profiles_refuse_new_work( + root: &std::path::Path, + state: &AppState, + engine: &std::sync::Arc, + fresh: &omnigraph_cluster::MaintenanceRequest, + retained: Option<(&omnigraph_cluster::MaintenanceRequest, &Value)>, +) { + fn submit(request: &omnigraph_cluster::MaintenanceRequest, token: &str) -> Request { + Request::post("/cluster/maintenance") + .header(HTTP_API_CONTRACT_HEADER, HTTP_API_CONTRACT) + .header("authorization", format!("Bearer {token}")) + .header("content-type", "application/json") + .body(Body::from(serde_json::to_vec(request).unwrap())) + .unwrap() + } + + let mut signed = data_tokens::DataTokens::new(); + let canonical_root = format!("file://{}", fs::canonicalize(root).unwrap().display()); + signed.document["canonical_root"] = canonical_root.clone().into(); + signed.trust = omnigraph_server::data_tokens::DataTokenTrust::from_json( + &serde_json::to_vec(&signed.document).unwrap(), + &canonical_root, + ) + .unwrap(); + let oidc_trust = oidc::trust(root); + let before_journal = fs::read(root.join("__cluster/maintenance.json")).ok(); + let before_authority = fs::read(root.join("__cluster/state.json")).unwrap(); + let before_head = engine.list_commits(None).await.unwrap()[0] + .graph_commit_id + .clone(); + + for use_oidc in [false, true] { + let attach_trust = |state: AppState| { + if use_oidc { + state.with_oidc_identity_trust(oidc_trust.clone()) + } else { + state.with_data_token_trust(signed.trust.clone()) + } + }; + let identity_token = if use_oidc { + oidc::token("alice", oidc::RESOURCE) + } else { + signed.identity_token() + }; + for limits_first in [false, true] { + let bounded = if limits_first { + attach_trust( + state + .clone() + .with_maintenance_limits(fresh.limits.clone()) + .unwrap(), + ) + } else { + attach_trust(state.clone()) + .with_maintenance_limits(fresh.limits.clone()) + .unwrap() + }; + let app = build_app(bounded); + for token in ["operator-token", identity_token.as_str()] { + for request in [ + get_request("/cluster/maintenance?graph=knowledge", token), + submit(fresh, token), + ] { + let (code, body) = json_response(&app, request).await; + assert_eq!(code, StatusCode::CONFLICT, "{body}"); + assert!( + body.to_string().contains("resource capacity admission"), + "{body}" + ); + } + } + if let Some((request, receipt)) = retained { + for read in [ + get_request( + &format!( + "/cluster/maintenance/{}?graph=knowledge", + request.operation_id + ), + "operator-token", + ), + submit(request, "operator-token"), + ] { + let (code, body) = json_response(&app, read).await; + assert_eq!(code, StatusCode::OK, "{body}"); + assert_eq!(&body["maintenance"]["result"], receipt); + assert_eq!(body["in_progress"], false); + } + } + assert_eq!( + fs::read(root.join("__cluster/maintenance.json")).ok(), + before_journal + ); + assert_eq!( + fs::read(root.join("__cluster/state.json")).unwrap(), + before_authority + ); + assert_eq!( + engine.list_commits(None).await.unwrap()[0].graph_commit_id, + before_head + ); + } + } +} + async fn live_deployment_fixture(mode: &str) { use omnigraph_cluster::{CapturedDeployment, DeploymentStatus}; use omnigraph_server::{GraphId, GraphKey, RegistryLookup, ServerConfigMode}; @@ -2650,6 +3415,26 @@ async fn live_deployment_fixture(mode: &str) { let policy = fs::read_to_string(temp.path().join("policy.yaml")).unwrap(); fs::write(temp.path().join("policy.yaml"), policy.replace("[schema_apply, read, invoke_query]", "[schema_apply, read, invoke_query, change, branch_create, branch_merge, branch_delete]")).unwrap(); } + if mode == "maintenance" { + let path = temp.path().join("policy.yaml"); + let policy = fs::read_to_string(&path).unwrap(); + fs::write( + path, + policy + .replace( + "[schema_apply, read, invoke_query]", + "[schema_apply, read, invoke_query, change, optimize]", + ) + .replace( + "admins: [operator]", + &format!( + "admins: [operator, \"{}\", principal:stable_alice]", + data_tokens::DataTokens::new().actor + ), + ), + ) + .unwrap(); + } apply_cluster_fixture(temp.path()).await; let mut settings = cluster_settings(temp.path()).await.unwrap(); settings @@ -2690,6 +3475,10 @@ async fn live_deployment_fixture(mode: &str) { panic!("ready") }; let original_engine = Arc::clone(&original.engine); + if mode == "maintenance" { + served_optimization_journey(temp.path(), &state, &original_engine).await; + return; + } let original_contract = original_engine.schema_contract_digest(); let before = fs::read(temp.path().join("__cluster/state.json")).unwrap(); let boot_ledger: serde_json::Value = serde_json::from_slice(&before).unwrap(); diff --git a/crates/omnigraph-server/tests/integration/openapi.rs b/crates/omnigraph-server/tests/integration/openapi.rs index b2a29aa7e..2f5987ba9 100644 --- a/crates/omnigraph-server/tests/integration/openapi.rs +++ b/crates/omnigraph-server/tests/integration/openapi.rs @@ -202,6 +202,8 @@ const EXPECTED_PATHS: &[&str] = &[ "/cluster/plan", "/cluster/deployments", "/cluster/deployments/{deployment_id}", + "/cluster/maintenance", + "/cluster/maintenance/{operation_id}", "/graphs/{graph_id}/snapshot", "/graphs/{graph_id}/blob", "/graphs/{graph_id}/query", @@ -266,6 +268,203 @@ fn openapi_has_no_unexpected_paths() { } } +#[test] +fn openapi_maintenance_exposes_exact_submission_and_observation_fields() { + let doc = openapi_json(); + let paths = &doc["paths"]; + let schemas = &doc["components"]["schemas"]; + let submit = &paths["/cluster/maintenance"]["post"]; + assert_eq!(submit["operationId"], "optimize"); + assert_eq!( + submit["requestBody"]["content"]["application/json"]["schema"]["$ref"], + "#/components/schemas/MaintenanceRequest" + ); + let request = &schemas["MaintenanceRequest"]; + for field in ["operation_id", "graph", "kind", "limits"] { + assert!( + request["properties"].get(field).is_some(), + "{field}: {request}" + ); + assert!( + request["required"] + .as_array() + .unwrap() + .contains(&Value::from(field)) + ); + } + assert_eq!(request["additionalProperties"], false); + assert_eq!( + request["properties"]["kind"]["$ref"], + "#/components/schemas/MaintenanceKind" + ); + assert_eq!( + schemas["MaintenanceKind"]["enum"], + serde_json::json!(["optimize"]) + ); + assert_eq!( + request["properties"]["limits"]["$ref"], + "#/components/schemas/MaintenanceLimits" + ); + let limits = &schemas["MaintenanceLimits"]; + for field in [ + "max_tables", + "max_fragments", + "max_rows", + "max_source_bytes", + "max_files", + ] { + assert_eq!(limits["properties"][field]["minimum"].as_f64(), Some(1.0)); + assert!( + limits["required"] + .as_array() + .unwrap() + .contains(&Value::from(field)) + ); + } + assert_eq!(limits["additionalProperties"], false); + assert_eq!( + schemas["MaintenanceStatusOutput"]["properties"]["limits"]["$ref"], + "#/components/schemas/MaintenanceLimits" + ); + assert_eq!( + schemas["MaintenanceStatusOutput"]["properties"]["status"]["$ref"], + "#/components/schemas/MaintenanceStatus" + ); + for field in ["ledger_id", "next_sequence", "outstanding_id"] { + assert!( + schemas["MaintenanceStatus"]["required"] + .as_array() + .unwrap() + .contains(&Value::from(field)) + ); + } + for (path, operation_id) in [ + ("/cluster/maintenance", "maintenanceStatus"), + ("/cluster/maintenance/{operation_id}", "maintenanceLookup"), + ] { + let operation = &paths[path]["get"]; + assert_eq!(operation["operationId"], operation_id); + assert!( + operation["parameters"] + .as_array() + .unwrap() + .iter() + .any(|param| param["name"] == "graph" + && param["in"] == "query" + && param["required"] == true) + ); + } + let receipt = &schemas["MaintenanceOutput"]["properties"]["maintenance"]; + assert_eq!(receipt["$ref"], "#/components/schemas/MaintenanceLookup"); + let variants = schemas["MaintenanceLookup"]["oneOf"].as_array().unwrap(); + let tags: HashSet<_> = variants + .iter() + .map(|variant| variant["properties"]["status"]["enum"][0].as_str().unwrap()) + .collect(); + assert_eq!( + tags, + HashSet::from([ + "outstanding", + "complete", + "result_expired", + "not_recorded", + "identity_mismatch", + "different_ledger", + ]) + ); + let results = schemas["MaintenanceResult"]["oneOf"].as_array().unwrap(); + for (tag, fields) in [ + ( + "success", + vec!["operation_id", "graph", "invocation", "report", "outcome"], + ), + ( + "refused", + vec![ + "operation_id", + "graph", + "invocation", + "code", + "message", + "outcome", + ], + ), + ] { + let variant = results + .iter() + .find(|value| value["properties"]["outcome"]["enum"][0] == tag) + .unwrap(); + for field in fields { + assert!( + variant["required"] + .as_array() + .unwrap() + .contains(&Value::from(field)), + "{tag}.{field}" + ); + } + } + for (name, fields) in [ + ( + "MaintenanceInvocation", + vec![ + "request", + "actor", + "canonical_root", + "result_revision", + "schema_contract", + "input", + ], + ), + ( + "OptimizeReport", + vec![ + "before_commit", + "after_commit", + "tables", + "source_tables", + "source_fragments", + "source_rows", + "source_bytes", + "source_files", + ], + ), + ( + "OptimizeTableStats", + vec![ + "type_key", + "fragments_removed", + "fragments_added", + "committed", + "skipped", + "published_dataset_version", + "lance_head_version", + "pending_indexes", + ], + ), + ] { + for field in fields { + assert!( + schemas[name]["required"] + .as_array() + .unwrap() + .contains(&Value::from(field)), + "{name}.{field}" + ); + } + } + assert_eq!( + schemas["MaintenanceOutput"]["example"]["maintenance"]["status"], + "not_recorded" + ); + assert!( + receipt["description"] + .as_str() + .unwrap() + .contains("never permits replay") + ); +} + #[test] fn openapi_commit_changes_is_get_with_page_token_params() { let doc = openapi_json(); @@ -2524,6 +2723,8 @@ async fn multi_mode_openapi_keeps_management_paths_flat() { "/cluster/plan", "/cluster/deployments", "/cluster/deployments/{deployment_id}", + "/cluster/maintenance", + "/cluster/maintenance/{operation_id}", "/.well-known/oauth-protected-resource", "/.well-known/oauth-protected-resource/{resource_path}", ] { @@ -2609,6 +2810,8 @@ async fn multi_mode_openapi_declares_graph_id_path_parameter() { "/cluster/plan", "/cluster/deployments", "/cluster/deployments/{deployment_id}", + "/cluster/maintenance", + "/cluster/maintenance/{operation_id}", "/.well-known/oauth-protected-resource", "/.well-known/oauth-protected-resource/{resource_path}", ] { diff --git a/crates/omnigraph/src/db/mod.rs b/crates/omnigraph/src/db/mod.rs index 85f3d3415..3470c03cd 100644 --- a/crates/omnigraph/src/db/mod.rs +++ b/crates/omnigraph/src/db/mod.rs @@ -21,13 +21,15 @@ pub use omnigraph::{ CollectorRowSummary, CommitListPage, DatasetCleanupStats, DatasetOptimizeStats, DatasetRepairStats, EXPORT_CHUNK_MAX_BYTES, ExportCut, FullTextIndexRebuildResult, GraphCreateReconciliation, InitOptions, MergeOutcome, MergeResult, Omnigraph, OpenMode, - PendingIndex, PreparedGraphCreate, PreparedSchemaApply, PreparedSchemaSettlement, ReadCheck, - RebuiltFullTextIndex, RepairAction, RepairClassification, RepairOptions, RepairStats, - RetainedManifestVersions, SYSTEM_COLUMNS_PREFLIGHT, SchemaApplyReconciliation, - SchemaApplyResult, SchemaApplySettlement, SchemaContractDigest, SchemaNonPublicationProof, - SkipReason, StagingVerdict, SystemColumnUpgradeFinding, SystemColumnUpgradeOptions, - SystemColumnUpgradeOutcome, SystemColumnUpgradeReport, TableCollectionPlan, - UnattachedTagRecord, UnpublishedManifest, unattached_release_order, + OptimizeFailure, OptimizeInput, OptimizeLimits, OptimizePublication, OptimizeReport, + OptimizeSettlement, OptimizeSettlementOutcome, PendingIndex, PreparedGraphCreate, + PreparedSchemaApply, PreparedSchemaSettlement, ReadCheck, RebuiltFullTextIndex, RepairAction, + RepairClassification, RepairOptions, RepairStats, RetainedManifestVersions, + SYSTEM_COLUMNS_PREFLIGHT, SchemaApplyReconciliation, SchemaApplyResult, SchemaApplySettlement, + SchemaContractDigest, SchemaNonPublicationProof, SkipReason, StagingVerdict, + SystemColumnUpgradeFinding, SystemColumnUpgradeOptions, SystemColumnUpgradeOutcome, + SystemColumnUpgradeReport, TableCollectionPlan, UnattachedTagRecord, UnpublishedManifest, + unattached_release_order, }; pub(crate) use omnigraph::{ LogicalBlobValue, RangedExternalBlobs, logical_row_image, row_blob_values, diff --git a/crates/omnigraph/src/db/omnigraph.rs b/crates/omnigraph/src/db/omnigraph.rs index 2a5b62ff7..7d229293a 100644 --- a/crates/omnigraph/src/db/omnigraph.rs +++ b/crates/omnigraph/src/db/omnigraph.rs @@ -33,6 +33,7 @@ use crate::table_store::TableStore; pub(crate) mod collector; mod export; pub(crate) mod optimize; +pub(crate) mod optimize_recovery; mod prepared_create; pub(crate) mod promotion; mod repair; @@ -50,7 +51,11 @@ pub use export::{EXPORT_CHUNK_MAX_BYTES, ExportCut}; pub(crate) use export::{ LogicalBlobValue, RangedExternalBlobs, logical_row_image, row_blob_values, }; -pub use optimize::{CleanupPolicyOptions, DatasetCleanupStats, DatasetOptimizeStats, SkipReason}; +pub use optimize::{ + CleanupPolicyOptions, DatasetCleanupStats, DatasetOptimizeStats, OptimizeFailure, + OptimizeInput, OptimizeLimits, OptimizePublication, OptimizeReport, SkipReason, +}; +pub use optimize_recovery::{OptimizeSettlement, OptimizeSettlementOutcome}; use prepared_create::initial_schema_ir; pub use prepared_create::{GraphCreateReconciliation, PreparedGraphCreate}; pub use repair::{ @@ -3586,7 +3591,86 @@ impl Omnigraph { /// Compact small Lance fragments into fewer larger ones across every /// backing dataset for a node or edge type on `main`. See [`optimize`] for details. pub async fn optimize(&self) -> Result> { - optimize::optimize_all_datasets(self).await + self.optimize_as(None).await + } + + /// Optimize `main` as an actor authorized for the `optimize` policy action. + pub async fn optimize_as( + &self, + actor: Option<&str>, + ) -> Result> { + let mut effects_started = false; + Ok(optimize::optimize_all_datasets( + self, + actor, + None, + &mut effects_started, + |_| async { Ok(()) }, + |_| async { Ok(()) }, + ) + .await? + .tables) + } + + /// Optimize `main` with finite source-inventory admission and one table in + /// flight. Refuse the complete attempt before any effects if accounting is + /// unsupported or a limit is exceeded. An effectful error is not replayable. + pub async fn optimize_with_limits_as( + &self, + limits: &OptimizeLimits, + actor: Option<&str>, + ) -> std::result::Result { + self.optimize_with_limits_admitted_as(limits, actor, |_| async { Ok(()) }) + .await + } + + /// Bind the accepted immutable input to an invocation before any physical + /// effect. The callback runs once with schema, main and table gates held; + /// it must not call graph operations that acquire those same gates. + pub async fn optimize_with_limits_admitted_as( + &self, + limits: &OptimizeLimits, + actor: Option<&str>, + admit: F, + ) -> std::result::Result + where + F: FnOnce(OptimizeInput) -> Fut, + Fut: std::future::Future>, + { + self.optimize_with_limits_recorded_as(limits, actor, admit, |_| async { Ok(()) }) + .await + } + + /// Bind input before effects and exact detached-table outcomes before + /// publication. The second callback runs only for productive work, with + /// the same native gates held; neither callback may reenter those gates. + pub async fn optimize_with_limits_recorded_as( + &self, + limits: &OptimizeLimits, + actor: Option<&str>, + admit: F, + record_publication: P, + ) -> std::result::Result + where + F: FnOnce(OptimizeInput) -> Fut, + Fut: std::future::Future>, + P: FnOnce(OptimizePublication) -> PubFut, + PubFut: std::future::Future>, + { + let mut effects_started = false; + optimize::optimize_all_datasets( + self, + actor, + Some(limits), + &mut effects_started, + admit, + record_publication, + ) + .await + .map_err(|error| OptimizeFailure { + error, + effects_started, + }) } /// Classify and explicitly repair uncovered graph-manifest/Lance-HEAD drift. See diff --git a/crates/omnigraph/src/db/omnigraph/optimize.rs b/crates/omnigraph/src/db/omnigraph/optimize.rs index dd5517308..c2d9f5c99 100644 --- a/crates/omnigraph/src/db/omnigraph/optimize.rs +++ b/crates/omnigraph/src/db/omnigraph/optimize.rs @@ -61,7 +61,8 @@ pub struct CleanupPolicyOptions { /// Why `optimize` did not compact a dataset. Typed so callers branch on the /// reason rather than sniffing a string. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] #[non_exhaustive] pub enum SkipReason { /// The Lance dataset HEAD is ahead of the version recorded in @@ -97,7 +98,7 @@ impl std::fmt::Display for SkipReason { /// Per-dataset outcome of `optimize_all_datasets`. This is a returned result type, /// not built by callers, so it is `#[non_exhaustive]`: future fields stay /// non-breaking and downstream code reads fields rather than constructing it. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[non_exhaustive] pub struct DatasetOptimizeStats { /// Qualified graph type key, or `__manifest` for the system dataset. @@ -125,6 +126,126 @@ pub struct DatasetOptimizeStats { pub pending_indexes: Vec, } +/// Admission ceilings for one optimization of the accepted `main` snapshot. +/// These bound source inventory, not peak memory, output bytes or total I/O. +/// Bounded optimization always executes one table at a time. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OptimizeLimits { + /// Accepted data tables plus the native `__manifest` catalog table. + pub max_tables: u32, + /// Physical fragments, including `__manifest` fragments. + pub max_fragments: u64, + /// Physical data rows plus native catalog records, including deleted rows. + pub max_rows: u64, + pub max_source_bytes: u64, + pub max_files: u64, +} + +impl OptimizeLimits { + /// Refuse missing bounds before accepting or executing an operation. + pub fn validate(&self) -> Result<()> { + for (name, value) in [ + ("tables", u64::from(self.max_tables)), + ("fragments", self.max_fragments), + ("rows", self.max_rows), + ("source_bytes", self.max_source_bytes), + ("files", self.max_files), + ] { + if value == 0 { + return Err(OmniError::manifest(format!( + "optimize limit {name} must be nonzero" + ))); + } + } + Ok(()) + } +} + +/// Immutable input for one bounded attempt. Source counts include `__manifest`: +/// `source_tables` counts the catalog table and `source_rows` its native records, +/// as well as the accepted data tables and their physical rows. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OptimizeInput { + pub before_commit: String, + /// Actual accepted graph schema, which can differ from applied cluster + /// configuration when the graph schema was changed independently. + pub schema_contract: SchemaContractDigest, + /// Conservative compact-JSON byte bound for the complete exact report. + /// Derived from the accepted metadata before effects, independently of + /// source-row sizes; the invocation owner reserves its own envelope too. + pub report_max_bytes: u64, + pub source_tables: u32, + pub source_fragments: u64, + pub source_rows: u64, + pub source_bytes: u64, + pub source_files: u64, +} + +/// Exact publication and conservative source inventory for one bounded attempt. +/// Source counts have the same catalog-inclusive meaning as [`OptimizeInput`]. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OptimizeReport { + pub before_commit: String, + pub after_commit: String, + pub tables: Vec, + pub source_tables: u32, + pub source_fragments: u64, + pub source_rows: u64, + pub source_bytes: u64, + pub source_files: u64, +} + +/// Exact table outcomes captured after detached work and before the one graph +/// publication. A durable owner can resolve publication by this unique intent +/// nonce after process exclusion; this does not claim the derived tail finished. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OptimizePublication { + pub before_manifest_version: u64, + pub intent_nonce: String, + pub schema_contract: SchemaContractDigest, + pub actor: Option, + pub created_at: i64, + pub report: OptimizeReport, +} + +/// Whether a failed optimization may have produced physical effects. Callers +/// may record an effect-free refusal; an effectful failure is not replayable. +#[derive(Debug, thiserror::Error)] +#[error("{error}")] +pub struct OptimizeFailure { + #[source] + pub error: OmniError, + pub effects_started: bool, +} + +#[derive(Default)] +struct OptimizeSources { + tables: u32, + fragments: u64, + rows: u64, + bytes: u64, + files: u64, +} + +fn add_source(total: &mut u64, amount: u64, limit: u64, name: &str) -> Result<()> { + let actual = total + .checked_add(amount) + .ok_or_else(|| OmniError::manifest(format!("optimize source {name} count overflow")))?; + if actual > limit { + return Err(OmniError::ResourceLimitExceeded { + resource: format!("optimize_source_{name}"), + limit, + actual, + }); + } + *total = actual; + Ok(()) +} + impl DatasetOptimizeStats { /// Stat for a dataset that Lance actually compacted. fn compacted(type_key: String, metrics: &CompactionMetrics, committed: bool) -> Self { @@ -179,6 +300,7 @@ struct PreparedOptimizeTable { initial_snapshot: crate::storage_layer::SnapshotHandle, witness: crate::table_store::StagingWitness, last_linear_version: Option, + report_pending_bound: Vec, } enum OptimizePreparation { @@ -216,7 +338,28 @@ decide_seam! { /// with an exact pin CAS, so one public Optimize produces at most one graph /// commit. The publisher owns main's `__manifest` fragment layout; maintenance /// only removes obsolete automatic-cleanup configuration from that dataset. -pub async fn optimize_all_datasets(db: &Omnigraph) -> Result> { +pub async fn optimize_all_datasets( + db: &Omnigraph, + actor: Option<&str>, + limits: Option<&OptimizeLimits>, + effects_started: &mut bool, + admit: F, + record_publication: P, +) -> Result +where + F: FnOnce(OptimizeInput) -> Fut, + Fut: std::future::Future>, + P: FnOnce(OptimizePublication) -> PubFut, + PubFut: std::future::Future>, +{ + db.enforce( + omnigraph_policy::PolicyAction::Optimize, + &omnigraph_policy::ResourceScope::TargetBranch("main".to_string()), + actor, + )?; + if let Some(limits) = limits { + limits.validate()?; + } let _export_exclusion = db.reserve_export_destructive_control()?; db.ensure_schema_state_valid().await?; @@ -246,6 +389,11 @@ pub async fn optimize_all_datasets(db: &Omnigraph) -> Result Result>(); - let concurrency = maint_concurrency().min(table_tasks.len()).max(1); - - let preparations: Vec> = futures::stream::iter(table_tasks) - .map(|task| { - let catalog = std::sync::Arc::clone(&catalog); - async move { prepare_optimize_table(db, catalog.as_ref(), task).await } - }) - .buffer_unordered(concurrency) - .collect() - .await; + let mut sources = OptimizeSources::default(); + let preparations = if let Some(limits) = limits { + let tables = u64::try_from(table_tasks.len()) + .ok() + .and_then(|tables| tables.checked_add(1)) + .ok_or_else(|| OmniError::manifest("optimize table count overflow"))?; + if tables > u64::from(limits.max_tables) { + return Err(OmniError::ResourceLimitExceeded { + resource: "optimize_source_tables".to_string(), + limit: u64::from(limits.max_tables), + actual: tables, + }); + } + sources.tables = tables as u32; + // The publication and stale-config repair also consume __manifest's + // accepted metadata and physical files. Pin the same version as the + // revalidated authority; never inventory a later HEAD. + let manifest = crate::instrumentation::open_dataset( + &crate::db::manifest::manifest_uri(db.root_uri()), + crate::instrumentation::VersionResolution::At(snapshot.graph_manifest_version()), + Some(&db.control_session()), + crate::instrumentation::manifest_wrapper(), + ) + .await?; + account_optimize_sources(&manifest, &mut sources, limits).await?; + let mut preparations = Vec::with_capacity(table_tasks.len()); + for task in table_tasks { + // Account every accepted table before the first effect. A later + // table exceeding the budget cannot leave an earlier one rewritten. + preparations.push(Ok(prepare_optimize_table( + db, + catalog.as_ref(), + task, + Some((&mut sources, limits)), + ) + .await?)); + } + preparations + } else { + let concurrency = maint_concurrency().min(table_tasks.len()).max(1); + futures::stream::iter(table_tasks) + .map(|task| { + let catalog = std::sync::Arc::clone(&catalog); + async move { prepare_optimize_table(db, catalog.as_ref(), task, None).await } + }) + .buffer_unordered(concurrency) + .collect::>>() + .await + }; let mut prepared = Vec::new(); let mut stats = Vec::new(); @@ -288,20 +475,55 @@ pub async fn optimize_all_datasets(db: &Omnigraph) -> Result> = futures::stream::iter(prepared) - .map(|work| { - let catalog = std::sync::Arc::clone(&catalog); - async move { apply_optimize_table_effects(db, catalog.as_ref(), work).await } - }) - .buffer_unordered(effect_concurrency) - .collect() - .await; - let mut outcomes = Vec::new(); - for result in effect_results { - outcomes.push(result?); - } + *effects_started = true; + let mut outcomes: Vec = if limits.is_some() { + // One table in flight: stop before starting the next table if this + // one's effects fail or become uncertain. + let mut outcomes = Vec::with_capacity(prepared.len()); + for work in prepared { + outcomes.push(apply_optimize_table_effects(db, catalog.as_ref(), work).await?); + } + outcomes + } else { + // Retain the direct path's settlement of all in-flight work. + let effect_concurrency = maint_concurrency().min(prepared.len()).max(1); + futures::stream::iter(prepared) + .map(|work| { + let catalog = std::sync::Arc::clone(&catalog); + async move { apply_optimize_table_effects(db, catalog.as_ref(), work).await } + }) + .buffer_unordered(effect_concurrency) + .collect::>() + .await + .into_iter() + .collect::>()? + }; fail(&OPTIMIZE_POST_PHASE_B_PRE_MANIFEST_COMMIT)?; let mut updates = Vec::new(); let mut expected_versions = crate::db::manifest::ExpectedTableVersions::new(); @@ -324,19 +546,49 @@ pub async fn optimize_all_datasets(db: &Omnigraph) -> Result Result>>(); - all.push(clear_manifest_auto_cleanup(db).await); - - if let Err(error) = persist_graph_index_artifact(db, &catalog, edge_tables_committed).await { + let manifest_stat = clear_manifest_auto_cleanup(db, effects_started).await; + all.push(if limits.is_some() { + Ok(manifest_stat?) + } else { + manifest_stat + }); + + if let Err(error) = persist_graph_index_artifact( + db, + &catalog, + edge_tables_committed, + bounded_snapshot.as_ref(), + effects_started, + ) + .await + { + if limits.is_some() { + return Err(error); + } tracing::warn!( target: "omnigraph::optimize", error = %error, @@ -363,7 +650,17 @@ pub async fn optimize_all_datasets(db: &Omnigraph) -> Result>()?, + source_tables: sources.tables, + source_fragments: sources.fragments, + source_rows: sources.rows, + source_bytes: sources.bytes, + source_files: sources.files, + }) } /// Build the full-catalog graph index from a fresh main snapshot and write it @@ -386,6 +683,8 @@ async fn persist_graph_index_artifact( db: &Omnigraph, catalog: &omnigraph_compiler::catalog::Catalog, edge_tables_committed: bool, + bounded_snapshot: Option<&crate::db::Snapshot>, + effects_started: &mut bool, ) -> Result<()> { if catalog.edge_types.is_empty() { return Ok(()); @@ -403,7 +702,10 @@ async fn persist_graph_index_artifact( .iter() .map(|(name, et)| (name.clone(), (et.from_type.clone(), et.to_type.clone()))) .collect(); - let snapshot = db.snapshot_for_branch(None).await?; + let snapshot = match bounded_snapshot { + Some(snapshot) => snapshot.clone(), + None => db.snapshot_for_branch(None).await?, + }; // `save` refuses a partially-materialized store (no complete identity to // stamp); check that BEFORE paying the full-catalog edge scan. if edge_types @@ -417,8 +719,9 @@ async fn persist_graph_index_artifact( return Ok(()); } let index = - crate::graph_index::GraphIndex::build(&snapshot, &edge_types, db.catalog().system_columns) + crate::graph_index::GraphIndex::build(&snapshot, &edge_types, catalog.system_columns) .await?; + *effects_started = true; let written = crate::graph_index::persist::save(&snapshot, db.storage_adapter(), &edge_types, &index) .await?; @@ -437,17 +740,141 @@ async fn persist_graph_index_artifact( Ok(()) } +/// Account the complete pinned physical inventory, not only Lance's chosen +/// compaction tasks. Index folding can scan a whole table and the adapter adds +/// dropped-column rewrites after Lance plans. Exact object HEADs fill missing +/// data-file sizes; no history or directory inventory is listed. +async fn account_optimize_sources( + ds: &lance::Dataset, + sources: &mut OptimizeSources, + limits: &OptimizeLimits, +) -> Result<()> { + let unsupported = |reason: &str| { + OmniError::manifest(format!( + "bounded optimize cannot account for {}: {reason}", + ds.uri() + )) + }; + let store = ds.object_store(None).await.map_err(OmniError::storage)?; + let base = ds.branch_location().path; + add_source(&mut sources.files, 1, limits.max_files, "files")?; + let manifest_bytes = match ds.manifest_location().size { + Some(bytes) => bytes, + None => store + .size(&ds.manifest_location().path) + .await + .map_err(OmniError::storage)?, + }; + add_source( + &mut sources.bytes, + manifest_bytes, + limits.max_source_bytes, + "bytes", + )?; + for fragment in ds.manifest().fragments.iter() { + add_source(&mut sources.fragments, 1, limits.max_fragments, "fragments")?; + let rows = fragment + .physical_rows + .ok_or_else(|| unsupported("unknown physical row count"))?; + add_source(&mut sources.rows, rows as u64, limits.max_rows, "rows")?; + if !fragment.overlays.is_empty() { + return Err(unsupported("data overlays")); + } + for file in &fragment.files { + if file.base_id.is_some() { + return Err(unsupported("inherited data-file storage base")); + } + add_source(&mut sources.files, 1, limits.max_files, "files")?; + let bytes = match file.file_size_bytes.get() { + Some(bytes) => bytes.get(), + None => store + .size(&base.clone().join("data").join(file.path.as_str())) + .await + .map_err(OmniError::storage)?, + }; + add_source(&mut sources.bytes, bytes, limits.max_source_bytes, "bytes")?; + } + if let Some(deletion) = &fragment.deletion_file { + if deletion.base_id.is_some() { + return Err(unsupported("inherited deletion-file storage base")); + } + add_source(&mut sources.files, 1, limits.max_files, "files")?; + let path = lance_table::io::deletion::deletion_file_path(&base, fragment.id, deletion); + let bytes = store.size(&path).await.map_err(OmniError::storage)?; + add_source(&mut sources.bytes, bytes, limits.max_source_bytes, "bytes")?; + } + } + // The query-facing listing filters indexes this reader cannot use. Such + // filtering is not evidence that their physical inputs are absent. + let indexes = lance_table::io::manifest::read_manifest_indexes( + &store, + ds.manifest_location(), + ds.manifest(), + ) + .await + .map_err(OmniError::storage)?; + for index in &indexes { + if index.base_id.is_some() || lance_index::is_system_index(index) { + return Err(unsupported("inherited or system index")); + } + if index.index_details.is_none() { + return Err(unsupported("unknown index-kind metadata")); + } + let files = index + .files + .as_ref() + .ok_or_else(|| unsupported("unknown index-file inventory"))?; + for file in files { + add_source(&mut sources.files, 1, limits.max_files, "files")?; + add_source( + &mut sources.bytes, + file.size_bytes, + limits.max_source_bytes, + "bytes", + )?; + } + } + let readable = ds.load_indices().await.map_err(OmniError::storage)?; + let readable_ids = readable + .iter() + .map(|index| index.uuid) + .collect::>(); + if indexes + .iter() + .any(|index| !readable_ids.contains(&index.uuid)) + { + return Err(unsupported("unsupported physical index format")); + } + // Descriptor scans do not hydrate Blob payloads. Charge every live managed + // value conservatively, including inline values already charged above. + let (bytes, files) = TableStore::managed_blob_source_size( + ds, + limits.max_source_bytes - sources.bytes, + limits.max_files - sources.files, + ) + .await?; + add_source(&mut sources.bytes, bytes, limits.max_source_bytes, "bytes")?; + add_source(&mut sources.files, files, limits.max_files, "files")?; + Ok(()) +} + /// Pure planning: classify drift/no-work/productive state without advancing /// any Lance HEAD. The caller holds schema -> main -> all table gates. async fn prepare_optimize_table( db: &Omnigraph, catalog: &omnigraph_compiler::catalog::Catalog, task: OptimizeTableTask, + accounting: Option<(&mut OptimizeSources, &OptimizeLimits)>, ) -> Result { + let bounded = accounting.is_some(); let snapshot = db .open_pinned_for_write(&task.full_path, &task.entry) .await?; + if let Some((sources, limits)) = accounting { + account_optimize_sources(snapshot.dataset(), sources, limits).await?; + } + // The same plan `stage_compaction` executes: a fragment still holding a // dropped property's values is compaction work even when Lance's planner // would leave the table alone. @@ -473,6 +900,11 @@ async fn prepare_optimize_table( return Ok(OptimizePreparation::Stat(stat)); } + let report_pending_bound = if bounded { + pending_index_report_bound(catalog, &task.table_key, &snapshot).await? + } else { + Vec::new() + }; Ok(OptimizePreparation::Work(PreparedOptimizeTable { identity: task.identity, table_key: task.table_key, @@ -481,9 +913,160 @@ async fn prepare_optimize_table( initial_snapshot: snapshot, witness: task.witness, last_linear_version: task.entry.version_metadata.last_linear_version(), + report_pending_bound, })) } +fn deferred_full_text_index(table_key: &str, property: &str) -> super::PendingIndex { + super::PendingIndex { + type_key: table_key.to_string(), + property: property.to_string(), + reason: "full-text index has incomplete or unknown coverage; \ + run omnigraph rebuild-full-text-indexes --branch main" + .to_string(), + } +} + +fn deferred_vector_fold_reason(reason: &str) -> String { + format!("vector index coverage lags and the column cannot train an index: {reason}") +} + +/// Every possible pending row has metadata known before physical work: +/// declared missing vectors, one vector-fold row per existing index name, +/// and full-text fields on existing segments or new declared columns. Counting +/// every segment deliberately over-reserves duplicate names. Compaction and +/// index folding preserve content and schema; they introduce no new properties. +async fn pending_index_report_bound( + catalog: &omnigraph_compiler::catalog::Catalog, + table_key: &str, + snapshot: &crate::storage_layer::SnapshotHandle, +) -> Result> { + let mut pending = Vec::new(); + if let Some(node) = table_key + .strip_prefix("node:") + .and_then(|name| catalog.node_types.get(name)) + { + for columns in &node.indices { + if columns.len() == 1 + && node.properties.get(&columns[0]).is_some_and(|property| { + property.index_kind() == Some(omnigraph_compiler::types::PropIndexKind::Vector) + }) + { + pending.push(super::PendingIndex { + type_key: table_key.to_string(), + property: columns[0].clone(), + reason: super::PendingIndex::NO_VECTORS_REASON.to_string(), + }); + } + } + } + let ds = snapshot.dataset(); + for index in ds.load_indices().await.map_err(OmniError::storage)?.iter() { + if TableStore::index_is_vector(index) { + if let Some(field) = index + .fields + .first() + .and_then(|id| ds.schema().field_by_id(*id)) + { + pending.push(super::PendingIndex { + type_key: table_key.to_string(), + property: field.name.clone(), + reason: deferred_vector_fold_reason(super::PendingIndex::NO_VECTORS_REASON), + }); + } + } else if is_full_text_index(index) { + for field in index + .keyed_fields() + .iter() + .filter_map(|id| ds.schema().field_by_id(*id)) + { + pending.push(deferred_full_text_index(table_key, &field.name)); + } + } + } + // A detached link can install a declaration for a previously absent FTS + // segment. Its generated physical name is irrelevant to the public reason. + for column in super::table_ops::declared_full_text_columns(catalog, table_key) { + pending.push(deferred_full_text_index(table_key, &column)); + } + Ok(pending) +} + +fn maximum_table_report( + type_key: String, + pending_indexes: Vec, +) -> DatasetOptimizeStats { + DatasetOptimizeStats { + type_key, + fragments_removed: usize::MAX, + fragments_added: usize::MAX, + committed: false, // JSON false is one byte longer than true. + skipped: Some(SkipReason::DriftNeedsRepair), + published_dataset_version: Some(u64::MAX), + lance_head_version: Some(u64::MAX), + pending_indexes, + } +} + +fn compact_json_bytes(value: &impl serde::Serialize) -> Result { + #[derive(Default)] + struct Counter(u64); + impl std::io::Write for Counter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0 = self + .0 + .checked_add(bytes.len() as u64) + .ok_or_else(|| std::io::Error::other("optimize report byte count overflow"))?; + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let mut counter = Counter::default(); + serde_json::to_writer(&mut counter, value) + .map_err(|error| OmniError::manifest(format!("encode optimize report bound: {error}")))?; + Ok(counter.0) +} + +fn optimize_report_byte_bound( + before_commit: &str, + sources: &OptimizeSources, + stats: &[DatasetOptimizeStats], + prepared: &[PreparedOptimizeTable], +) -> Result { + use omnigraph_core::graph_commit_id::{HISTORY_BLOCK_SLOTS, HistoryBlockId}; + // A productive publish emits a canonical ULID or history-block ID. A + // no-op retains the exact input spelling, including accepted older IDs. + let largest_commit = HistoryBlockId::new( + ulid::Ulid::nil(), + HISTORY_BLOCK_SLOTS - 1, + ulid::Ulid::nil(), + )? + .to_string(); + let after_commit = if compact_json_bytes(&before_commit)? > compact_json_bytes(&largest_commit)? + { + before_commit.to_string() + } else { + largest_commit + }; + let mut tables = stats.to_vec(); // No-work table reports are already exact. + tables.extend(prepared.iter().map(|work| { + maximum_table_report(work.table_key.clone(), work.report_pending_bound.clone()) + })); + tables.push(maximum_table_report("__manifest".to_string(), Vec::new())); + compact_json_bytes(&OptimizeReport { + before_commit: before_commit.to_string(), + after_commit, + tables, + source_tables: sources.tables, + source_fragments: sources.fragments, + source_rows: sources.rows, + source_bytes: sources.bytes, + source_files: sources.files, + }) +} + /// Deferred full-text coverage is observable status, never a promise that /// ordinary optimize will advance this table. Recheck after physical work too: /// stable-ID compaction can change coverage while preserving index artifacts. @@ -517,14 +1100,7 @@ async fn append_deferred_full_text_indexes( .iter() .filter_map(|id| ds.schema().field_by_id(*id)) { - pending.push(super::PendingIndex { - type_key: table_key.to_string(), - property: field.name.clone(), - reason: format!( - "full-text index '{name}' has incomplete or unknown coverage; \ - run omnigraph rebuild-full-text-indexes --branch main" - ), - }); + pending.push(deferred_full_text_index(table_key, &field.name)); } } Ok(()) @@ -597,9 +1173,7 @@ async fn apply_optimize_table_effects( index_work.pending.push(super::PendingIndex { type_key: table_key.clone(), property: column, - reason: format!( - "vector index coverage lags and the column cannot train an index: {reason}" - ), + reason: deferred_vector_fold_reason(&reason), }); } if !index_work.specs.is_empty() { @@ -670,6 +1244,7 @@ fn is_retryable_lance_conflict(err: &lance::Error) -> bool { /// Remove automatic GC configuration, returning whether Lance HEAD advanced. async fn clear_stale_auto_cleanup_config( ds: &mut lance::Dataset, + effects_started: &mut bool, ) -> std::result::Result { let keys: Vec = ds .config() @@ -684,17 +1259,21 @@ async fn clear_stale_auto_cleanup_config( // replacement for `delete_config_keys` (awaiting the builder merges rather // than replacing the whole config map). let entries: Vec<(&str, Option<&str>)> = keys.iter().map(|k| (k.as_str(), None)).collect(); + *effects_started = true; ds.update_config(entries).await?; Ok(true) } /// Remove stored automatic GC configuration without changing manifest fragments. -async fn clear_manifest_auto_cleanup(db: &Omnigraph) -> Result { +async fn clear_manifest_auto_cleanup( + db: &Omnigraph, + effects_started: &mut bool, +) -> Result { let uri = crate::db::manifest::manifest_uri(db.root_uri()); for attempt in 0..INTERNAL_CONFIG_RETRY_BUDGET { let handle = db.storage().open_dataset_head(&uri, None).await?; let mut ds = handle.into_dataset(); - match clear_stale_auto_cleanup_config(&mut ds).await { + match clear_stale_auto_cleanup_config(&mut ds, effects_started).await { Ok(cleared) => { if cleared { db.coordinator.write().await.refresh().await?; @@ -1519,6 +2098,187 @@ mod tests { use super::*; use crate::loader::LoadMode; + #[test] + fn report_reservation_counts_escaped_metadata_and_numeric_widths() { + let pending = vec![deferred_full_text_index( + "node:\"\\\n東京", + &"long\"\\\nproperty".repeat(8192), + )]; + let maximum = maximum_table_report("node:\"\\\n東京".to_string(), pending); + let reserved = compact_json_bytes(&maximum).unwrap(); + assert_eq!(reserved, serde_json::to_vec(&maximum).unwrap().len() as u64); + assert!(reserved > maximum.pending_indexes[0].property.len() as u64); + for committed in [false, true] { + for skipped in [None, Some(SkipReason::DriftNeedsRepair)] { + for version in [None, Some(0), Some(u64::MAX)] { + let mut actual = maximum.clone(); + actual.committed = committed; + actual.skipped = skipped; + actual.published_dataset_version = version; + actual.lance_head_version = version; + for fragments in [0, 9, 10, usize::MAX] { + actual.fragments_removed = fragments; + actual.fragments_added = fragments; + assert!(compact_json_bytes(&actual).unwrap() <= reserved); + } + actual.pending_indexes.clear(); + assert!(compact_json_bytes(&actual).unwrap() <= reserved); + } + } + } + } + + #[tokio::test] + async fn bounded_inventory_refuses_unknown_rows_and_heads_unknown_file_sizes() { + let dir = tempfile::tempdir().unwrap(); + let db = crate::Session::from_defaults( + std::sync::Arc::new( + Omnigraph::init( + dir.path().to_str().unwrap(), + "node Person { name: String @key }", + ) + .await + .unwrap(), + ), + omnigraph_compiler::settings::SessionSettings::default(), + ); + db.load_jsonl( + "{\"type\":\"Person\",\"data\":{\"name\":\"Alice\"}}", + LoadMode::Overwrite, + ) + .await + .unwrap(); + let snapshot = db + .snapshot_of(crate::db::ReadTarget::branch("main")) + .await + .unwrap(); + let entry = snapshot.dataset("node:Person").unwrap(); + let path = format!("{}/{}", db.root_uri(), entry.dataset_path); + let mut ds = db + .open_pinned_for_write(&path, entry) + .await + .unwrap() + .into_dataset(); + let limits = OptimizeLimits { + max_tables: 1, + max_fragments: 16, + max_rows: 16, + max_source_bytes: 1024 * 1024, + max_files: 64, + }; + let mut known = OptimizeSources::default(); + account_optimize_sources(&ds, &mut known, &limits) + .await + .unwrap(); + let mut fragments = ds.manifest.fragments.as_ref().clone(); + for file in &mut fragments[0].files { + file.file_size_bytes = Default::default(); + } + std::sync::Arc::make_mut(&mut ds.manifest).fragments = + std::sync::Arc::new(fragments.clone()); + let mut headed = OptimizeSources::default(); + account_optimize_sources(&ds, &mut headed, &limits) + .await + .unwrap(); + assert_eq!( + known.bytes, headed.bytes, + "unknown file size must be resolved by exact HEAD" + ); + fragments[0].physical_rows = None; + std::sync::Arc::make_mut(&mut ds.manifest).fragments = std::sync::Arc::new(fragments); + assert!( + account_optimize_sources(&ds, &mut OptimizeSources::default(), &limits) + .await + .is_err(), + "unknown physical rows cannot be treated as zero" + ); + } + + #[tokio::test] + async fn bounded_inventory_refuses_indexes_hidden_by_the_reader() { + use lance::index::DatasetIndexExt; + + let dir = tempfile::tempdir().unwrap(); + let db = crate::Session::from_defaults( + std::sync::Arc::new( + Omnigraph::init( + dir.path().to_str().unwrap(), + "node Person { name: String @key }", + ) + .await + .unwrap(), + ), + omnigraph_compiler::settings::SessionSettings::default(), + ); + db.load_jsonl( + "{\"type\":\"Person\",\"data\":{\"name\":\"Alice\"}}", + LoadMode::Overwrite, + ) + .await + .unwrap(); + let snapshot = db + .snapshot_of(crate::db::ReadTarget::branch("main")) + .await + .unwrap(); + let entry = snapshot.dataset("node:Person").unwrap(); + let path = format!("{}/{}", db.root_uri(), entry.dataset_path); + let mut ds = db + .open_pinned_for_write(&path, entry) + .await + .unwrap() + .into_dataset(); + let mut future_index = ds + .create_index_builder( + &["name"], + lance_index::IndexType::BTree, + &lance_index::scalar::ScalarIndexParams::default(), + ) + .name("future-index".to_string()) + .execute_uncommitted() + .await + .unwrap(); + future_index.index_version = i32::MAX; + assert!( + future_index + .files + .as_ref() + .is_some_and(|files| !files.is_empty()) + ); + let transaction = lance::dataset::transaction::Transaction::new( + ds.version().version, + lance::dataset::transaction::Operation::CreateIndex { + new_indices: vec![future_index], + removed_indices: Vec::new(), + }, + None, + ); + let ds = lance::dataset::CommitBuilder::new(std::sync::Arc::new(ds)) + .with_detached(true) + .execute(transaction) + .await + .unwrap(); + assert!( + ds.load_indices() + .await + .unwrap() + .iter() + .all(|index| index.name != "future-index") + ); + let limits = OptimizeLimits { + max_tables: 16, + max_fragments: 1024, + max_rows: 100_000, + max_source_bytes: 64 * 1024 * 1024, + max_files: 4096, + }; + assert!( + account_optimize_sources(&ds, &mut OptimizeSources::default(), &limits) + .await + .is_err(), + "filtered reader metadata cannot prove complete physical inventory" + ); + } + /// Native commit conflicts allow configuration cleanup to reopen and retry. #[test] fn retryable_lance_conflicts_are_classified() { diff --git a/crates/omnigraph/src/db/omnigraph/optimize_recovery.rs b/crates/omnigraph/src/db/omnigraph/optimize_recovery.rs new file mode 100644 index 000000000..0bed65bc1 --- /dev/null +++ b/crates/omnigraph/src/db/omnigraph/optimize_recovery.rs @@ -0,0 +1,260 @@ +//! Exact original-publication reconciliation, never replay of physical work. +use super::optimize::OptimizePublication; +use super::*; +use crate::db::manifest::{GraphHeadExpectation, LineageIntent, PublishPrecondition}; +use omnigraph_core::graph_commit_id::{canonical_ulid, commit_id_answers}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +/// Persist this native token before attempting the neutral publication. It is +/// evidence identity, not an authorization capability or writer exclusion. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OptimizeSettlement { + original_digest: String, + lineage: LineageIntent, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum OptimizeSettlementOutcome { + Published { report: OptimizeReport }, + Unpublished, + Unknown, +} + +fn digest(original: &OptimizePublication) -> Result { + let bytes = serde_json::to_vec(original) + .map_err(|error| OmniError::manifest(format!("encode optimize witness: {error}")))?; + Ok(format!("{:x}", Sha256::digest(bytes))) +} + +fn validate(original: &OptimizePublication) -> Result<()> { + if original.before_manifest_version == 0 + || original.before_manifest_version == u64::MAX + || canonical_ulid(&original.intent_nonce).is_none() + || original.report.before_commit != original.report.after_commit + || commit_id_answers(&original.report.before_commit, &original.intent_nonce)? + { + return Err(OmniError::manifest("invalid optimize publication witness")); + } + Ok(()) +} + +async fn retained_outcome( + db: &Omnigraph, + original: &OptimizePublication, + settlement: &OptimizeSettlement, +) -> Result { + let session = db.control_session(); + let Some(base) = omnigraph_catalog::read_schema_publication_candidate_at_with_session( + db.uri(), + original.before_manifest_version, + Some(&original.report.before_commit), + None, + Some(&session), + ) + .await? + else { + return Ok(OptimizeSettlementOutcome::Unknown); + }; + validate_schema_contract_row(&base.contract)?; + if base.head.graph_commit_id != original.report.before_commit + || SchemaContractDigest::from_row(&base.contract) != original.schema_contract + { + return Ok(OptimizeSettlementOutcome::Unknown); + } + let Some(candidate) = omnigraph_catalog::read_schema_publication_candidate_at_with_session( + db.uri(), + original.before_manifest_version + 1, + Some(&original.intent_nonce), + Some(&settlement.lineage.graph_commit_id), + Some(&session), + ) + .await? + else { + return Ok(OptimizeSettlementOutcome::Unknown); + }; + validate_schema_contract_row(&candidate.contract)?; + if candidate.branch_identifier != base.branch_identifier + || SchemaContractDigest::from_row(&candidate.contract) != original.schema_contract + { + return Ok(OptimizeSettlementOutcome::Unknown); + } + if candidate.original.is_some() { + let commit = &candidate.head; + if !commit_id_answers(&commit.graph_commit_id, &original.intent_nonce)? + || commit.graph_manifest_version != original.before_manifest_version + 1 + || commit.parent_commit_id.as_deref() != Some(&original.report.before_commit) + || commit.actor_id != original.actor + || commit.created_at != original.created_at + || commit.merged_parent_commit_id.is_some() + || candidate.settlement.is_some() + { + return Err(OmniError::manifest( + "optimize publication differs from its retained witness", + )); + } + // Reopen every accepted pin before admitting a successor. The witness + // names a publication; journal bytes never replace graph authority. + let snapshot = db + .snapshot_at_graph_manifest_version(commit.graph_manifest_version) + .await?; + let (catalog, _) = db.accepted_catalog_for_snapshot(&snapshot).await?; + validate_bound_catalog_against_snapshot(&catalog, &snapshot)?; + for entry in snapshot.datasets() { + snapshot.open_dataset(&entry.type_key).await?; + } + let mut report = original.report.clone(); + report.after_commit = commit.graph_commit_id.clone(); + return Ok(OptimizeSettlementOutcome::Published { report }); + } + if candidate.settlement.is_some() { + let commit = &candidate.head; + if !commit_id_answers(&commit.graph_commit_id, &settlement.lineage.graph_commit_id)? + || commit.graph_manifest_version != original.before_manifest_version + 1 + || commit.parent_commit_id.as_deref() != Some(&original.report.before_commit) + || commit.actor_id != settlement.lineage.actor_id + || commit.created_at != settlement.lineage.created_at + || commit.merged_parent_commit_id.is_some() + { + return Err(OmniError::manifest( + "optimize settlement differs from its retained witness", + )); + } + } else if candidate.head.graph_manifest_version <= original.before_manifest_version { + if candidate.head != base.head { + return Ok(OptimizeSettlementOutcome::Unknown); + } + } else if candidate.head.parent_commit_id.as_deref() != Some(&original.report.before_commit) { + return Ok(OptimizeSettlementOutcome::Unknown); + } + // An occupied exact candidate excludes the original, including a delayed + // accepted CAS. The original is forbidden to rebase to a later version. + Ok(OptimizeSettlementOutcome::Unpublished) +} + +impl Omnigraph { + pub async fn prepare_optimize_settlement_as( + &self, + original: &OptimizePublication, + actor: Option<&str>, + ) -> Result { + self.enforce( + omnigraph_policy::PolicyAction::Optimize, + &omnigraph_policy::ResourceScope::TargetBranch("main".into()), + actor, + )?; + validate(original)?; + Ok(OptimizeSettlement { + original_digest: digest(original)?, + lineage: self + .new_lineage_intent_for_branch( + None, + actor, + omnigraph_catalog::HistoryReleaseBytes::PRODUCTION, + ) + .await?, + }) + } + + /// After external predecessor/cleanup exclusion and native admission, read + /// the original's exact candidate. If still empty, publish one neutral + /// competitor there using the durably retained settlement identity. + pub async fn settle_optimize_as( + &self, + original: &OptimizePublication, + settlement: &OptimizeSettlement, + actor: Option<&str>, + ) -> Result { + self.enforce( + omnigraph_policy::PolicyAction::Optimize, + &omnigraph_policy::ResourceScope::TargetBranch("main".into()), + actor, + )?; + if self.coordinator.read().await.current_branch().is_some() { + return Err(OmniError::manifest( + "optimize recovery requires a main-bound graph handle", + )); + } + validate(original)?; + if settlement.original_digest != digest(original)? + || canonical_ulid(&settlement.lineage.graph_commit_id).is_none() + || settlement.lineage.graph_commit_id == original.intent_nonce + || settlement.lineage.branch.is_some() + || settlement.lineage.merged_parent.is_some() + || commit_id_answers( + &original.report.before_commit, + &settlement.lineage.graph_commit_id, + )? + { + return Err(OmniError::manifest("invalid optimize settlement binding")); + } + let _schema = self.write_queue().acquire_schema_exclusive().await; + let _export = self.reserve_export_destructive_control()?; + let _branch = self.write_queue().acquire_branch(None).await; + let outcome = retained_outcome(self, original, settlement).await?; + if outcome != OptimizeSettlementOutcome::Unknown { + return Ok(outcome); + } + let Some(base) = omnigraph_catalog::read_schema_publication_candidate_at_with_session( + self.uri(), + original.before_manifest_version, + Some(&original.report.before_commit), + None, + Some(&self.control_session()), + ) + .await? + else { + return Ok(OptimizeSettlementOutcome::Unknown); + }; + if base.head.graph_commit_id != original.report.before_commit + || SchemaContractDigest::from_row(&base.contract) != original.schema_contract + { + return Ok(OptimizeSettlementOutcome::Unknown); + } + self.refresh_coordinator_only().await?; + let snapshot = self.coordinator.read().await.snapshot(); + let keys = snapshot + .datasets() + .map(|entry| (entry.type_key.clone(), entry.native_dataset_branch.clone())) + .collect::>(); + let _tables = self.write_queue().acquire_many(&keys).await; + self.refresh_coordinator_only().await?; + let mut coordinator = self.coordinator.write().await; + if coordinator.version() != original.before_manifest_version + || coordinator.exact_graph_head().as_deref() != Some(&original.report.before_commit) + || SchemaContractDigest::from_row( + &coordinator + .snapshot() + .read_schema_contract(self.uri()) + .await?, + ) != original.schema_contract + { + return Ok(OptimizeSettlementOutcome::Unknown); + } + let authority = GraphHeadExpectation::new( + None, + coordinator.branch_identifier().await?, + Some(original.report.before_commit.clone()), + ); + let result = coordinator + .commit_changes_with_intent_and_expected( + &[], + &crate::db::manifest::ExpectedTableVersions::new(), + settlement.lineage.clone(), + &PublishPrecondition::ExactGraphVersion { + authority, + version: original.before_manifest_version, + }, + ) + .await; + drop(coordinator); + let outcome = retained_outcome(self, original, settlement).await?; + if outcome == OptimizeSettlementOutcome::Unknown + && let Err(error) = result + { + return Err(error.without_pre_effect_evidence()); + } + Ok(outcome) + } +} diff --git a/crates/omnigraph/src/db/omnigraph/schema_apply/prepared.rs b/crates/omnigraph/src/db/omnigraph/schema_apply/prepared.rs index 563bd4617..212a4a3f5 100644 --- a/crates/omnigraph/src/db/omnigraph/schema_apply/prepared.rs +++ b/crates/omnigraph/src/db/omnigraph/schema_apply/prepared.rs @@ -14,7 +14,7 @@ pub struct SchemaContractDigest { } impl SchemaContractDigest { - pub(super) fn from_row(row: &SchemaContractRow) -> Self { + pub(crate) fn from_row(row: &SchemaContractRow) -> Self { Self { source_hash: source_hash(&row.source), schema_ir_hash: row.head.schema_ir_hash.clone(), diff --git a/crates/omnigraph/src/db/omnigraph/table_ops.rs b/crates/omnigraph/src/db/omnigraph/table_ops.rs index 68875383a..652acc20c 100644 --- a/crates/omnigraph/src/db/omnigraph/table_ops.rs +++ b/crates/omnigraph/src/db/omnigraph/table_ops.rs @@ -776,7 +776,7 @@ async fn plan_index_work_node( work.pending.push(PendingIndex { type_key: table_key.to_string(), property: prop_name.clone(), - reason: "property has no non-null vectors to train on yet".to_string(), + reason: PendingIndex::NO_VECTORS_REASON.to_string(), }); } } @@ -1015,13 +1015,17 @@ pub(super) async fn open_owned_dataset_for_branch_write( /// vectors can be retried once populated; an existing full-text index with /// incomplete coverage requires explicit rebuilding. `reason` names the remedy. /// Reads retain their unindexed fallback; this status alone is not a write. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct PendingIndex { pub type_key: String, pub property: String, pub reason: String, } +impl PendingIndex { + pub(crate) const NO_VECTORS_REASON: &str = "property has no non-null vectors to train on yet"; +} + pub(super) async fn build_indices_on_dataset( db: &Omnigraph, table_key: &str, @@ -1271,6 +1275,8 @@ pub(super) async fn commit_updates( pub(crate) enum Publication { /// Advance the branch head: the ordinary publish. Publish, + /// Bind a durable maintenance publication witness to one exact version. + PublishExactVersion(u64), /// Write an unpublished commit (RFC "Unpublished graph commits"): the /// branch head is unchanged, and a merge carries its held inputs. Propose(Option), @@ -1289,9 +1295,16 @@ async fn publish_prepared( publication: Publication, ) -> Result { match publication { - Publication::Publish => { - let precondition = - crate::db::manifest::PublishPrecondition::ExactGraphHead(expectation); + Publication::Publish | Publication::PublishExactVersion(_) => { + let precondition = match publication { + Publication::PublishExactVersion(version) => { + crate::db::manifest::PublishPrecondition::ExactGraphVersion { + authority: expectation, + version, + } + } + _ => crate::db::manifest::PublishPrecondition::ExactGraphHead(expectation), + }; Ok(coordinator .commit_changes_with_intent_and_expected( changes, diff --git a/crates/omnigraph/src/table_store.rs b/crates/omnigraph/src/table_store.rs index dad9e5fb0..b636c6043 100644 --- a/crates/omnigraph/src/table_store.rs +++ b/crates/omnigraph/src/table_store.rs @@ -2828,7 +2828,7 @@ impl TableStore { /// The partition count Lance reports for a vector index, or one when the /// statistics do not name it. - async fn vector_partition_count(ds: &Dataset, name: &str) -> usize { + async fn vector_partition_count(ds: &Dataset, name: &str) -> Result { fn first_num_partitions(value: &serde_json::Value) -> Option { match value { serde_json::Value::Object(map) => map @@ -2840,13 +2840,16 @@ impl TableStore { _ => None, } } - ds.index_statistics(name) + let statistics = ds + .index_statistics(name) .await - .ok() - .and_then(|stats| serde_json::from_str::(&stats).ok()) - .and_then(|stats| first_num_partitions(&stats)) + .map_err(OmniError::storage)?; + let statistics: serde_json::Value = serde_json::from_str(&statistics).map_err(|error| { + OmniError::manifest_internal(format!("invalid vector index statistics: {error}")) + })?; + Ok(first_num_partitions(&statistics) .filter(|partitions| *partitions >= 1) - .unwrap_or(1) + .unwrap_or(1)) } async fn foldable_index_lag(ds: &Dataset) -> Result> { @@ -2896,8 +2899,9 @@ impl TableStore { /// merges the delta and previous segments through a crate-private path /// and commits linearly; this rebuilds each lagging index whole under its /// name with the public builder, which leaves the same single segment - /// Lance's merge would. A vector index the builder cannot train (a column - /// with no non-null rows) is skipped and reported by column. + /// Lance's merge would. A vector column with no non-null rows is deferred + /// before building. Actual builder failures + /// propagate, because an I/O failure can follow partial artifact writes. pub async fn stage_index_fold(&self, ds: &Dataset) -> Result { let lag = Self::foldable_index_lag(ds).await?; if lag.is_empty() { @@ -2920,10 +2924,22 @@ impl TableStore { let mut ds_clone = ds.clone(); let columns = [column.as_str()]; let built = if item.vector { - // Keep the index's partition count: the engine builds one - // partition, but a partitioned index (Lance's split, or an - // explicit build) keeps its shape across folds. - let partitions = Self::vector_partition_count(ds, &item.name).await; + let mut scanner = ds.scan(); + scanner.filter_expr( + Expr::Column(datafusion::common::Column::new_unqualified(&column)) + .is_not_null(), + ); + let rows = scanner.count_rows().await.map_err(OmniError::storage)?; + if rows == 0 { + skipped.push(( + column, + crate::db::PendingIndex::NO_VECTORS_REASON.to_string(), + )); + continue; + } + // Request the existing partition count. Lance alone reduces + // it when fewer trainable vectors remain after deletions. + let partitions = Self::vector_partition_count(ds, &item.name).await?; let params = lance::index::vector::VectorIndexParams::ivf_flat(partitions, MetricType::L2); ds_clone @@ -2943,16 +2959,6 @@ impl TableStore { }; let new_idx = match built { Ok(index) => index, - Err(error) if item.vector => { - tracing::warn!( - index = item.name.as_str(), - column = column.as_str(), - error = %error, - "vector index fold skipped: the column cannot train an index" - ); - skipped.push((column, error.to_string())); - continue; - } Err(error) => { return Err(OmniError::storage_context( format!("stage_index_fold: fold index '{}' on '{column}'", item.name), @@ -4390,6 +4396,76 @@ impl TableStore { })) } + /// Conservative managed Blob source inventory, without reading payloads. + /// One nonempty descriptor is charged as one source reference, even when + /// several descriptors share a packed file. External references are not + /// hydrated by compaction and do not charge unrelated external objects. + pub(crate) async fn managed_blob_source_size( + ds: &Dataset, + max_bytes: u64, + max_files: u64, + ) -> Result<(u64, u64)> { + if nested_blob_field(ds.schema()).is_some() + || ds + .schema() + .fields + .iter() + .any(|field| field.is_blob() && !field.is_blob_v2()) + { + return Err(OmniError::manifest( + "bounded optimize cannot account for nested or non-v2 Blob fields", + )); + } + let columns = ds + .schema() + .fields + .iter() + .filter(|field| field.is_blob()) + .map(|field| field.name.as_str()) + .collect::>(); + if columns.is_empty() { + return Ok((0, 0)); + } + let mut stream = Self::scan_stream_with(ds, Some(&columns), None, None, false, |scanner| { + scanner.batch_size(1024); + Ok(()) + }) + .await?; + let mut bytes = 0_u64; + let mut files = 0_u64; + while let Some(batch) = stream.try_next().await.map_err(OmniError::storage)? { + for column in batch.columns() { + let descriptions = column + .as_any() + .downcast_ref::() + .ok_or_else(|| OmniError::blob_integrity("expected Blob descriptions"))?; + let decoder = BlobDescriptorDecoder::try_new(descriptions)?; + for row in 0..batch.num_rows() { + if let BlobDescriptor::Managed { length } = decoder.classify(row)? { + bytes = bytes.checked_add(length).ok_or_else(|| { + OmniError::manifest("optimize Blob byte count overflow") + })?; + files = files.checked_add(u64::from(length > 0)).ok_or_else(|| { + OmniError::manifest("optimize Blob reference count overflow") + })?; + for (name, actual, limit) in + [("bytes", bytes, max_bytes), ("files", files, max_files)] + { + if actual > limit { + return Err(OmniError::ResourceLimitExceeded { + resource: format!("optimize_blob_source_{name}"), + limit, + actual, + }); + } + } + } + } + } + } + Ok((bytes, files)) + } + /// Most managed Blob bytes in one row of `fragments`, from descriptors (externals count /// nothing); after an early stop, a lower bound that already derives one-row batches. /// `None` when a nested or non-Blob-v2 Blob field cannot be sized. @@ -7224,6 +7300,111 @@ mod tests { RecordBatch::try_new(schema, vec![col]).unwrap() } + #[tokio::test] + async fn vector_fold_defers_untrainable_input_but_propagates_storage_failure() { + use arrow_array::builder::{FixedSizeListBuilder, Float32Builder}; + + for case in ["storage-failure", "empty", "reduced"] { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().to_str().unwrap(); + let uri = format!("{root}/vectors"); + let batch = |vectors: &[Option<[f32; 2]>]| { + let mut values = FixedSizeListBuilder::new(Float32Builder::new(), 2); + for vector in vectors { + values.values().append_slice(&vector.unwrap_or([0.0; 2])); + values.append(vector.is_some()); + } + let values = values.finish(); + let schema = Arc::new(Schema::new(vec![Field::new( + "embedding", + values.data_type().clone(), + true, + )])); + RecordBatch::try_new(schema, vec![Arc::new(values)]).unwrap() + }; + let initial = batch(&[ + Some([1.0, 2.0]), + Some([3.0, 4.0]), + Some([5.0, 6.0]), + Some([7.0, 8.0]), + ]); + let reader = + arrow_array::RecordBatchIterator::new(vec![Ok(initial.clone())], initial.schema()); + let mut dataset = Dataset::write(reader, &uri, None).await.unwrap(); + dataset + .create_index_builder( + &["embedding"], + IndexType::Vector, + &lance::index::vector::VectorIndexParams::ivf_flat(4, MetricType::L2), + ) + .await + .unwrap(); + if case == "empty" { + dataset.delete("true").await.unwrap(); + } else if case == "reduced" { + dataset.delete("embedding[1] < 7").await.unwrap(); + assert_eq!(dataset.count_rows(None).await.unwrap(), 1); + } + let appended = batch(&[(case != "empty").then_some([9.0, 10.0])]); + let reader = arrow_array::RecordBatchIterator::new( + vec![Ok(appended.clone())], + appended.schema(), + ); + let dataset = Dataset::write( + reader, + &uri, + Some(WriteParams { + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + assert!( + TableStore::has_foldable_unindexed_fragments(&dataset) + .await + .unwrap() + ); + let version = dataset.version().version; + if case == "storage-failure" { + // Preserve the exact input manifests but make the native index + // path unusable. A genuine storage error must not be reported + // as an untrainable column with a successful optimization. + let indices = dir.path().join("vectors/_indices"); + std::fs::rename(&indices, dir.path().join("retained-indices")).unwrap(); + std::fs::write(indices, b"not a directory").unwrap(); + } + let store = TableStore::new(root, Arc::new(lance::session::Session::default())); + let result = store.stage_index_fold(&dataset).await; + if case == "storage-failure" { + assert!( + result.is_err(), + "storage failure must propagate, never become pending" + ); + } else if case == "empty" { + let result = result.unwrap(); + assert!(result.staged.is_none()); + assert_eq!(result.skipped.len(), 1); + assert_eq!(result.skipped[0].0, "embedding"); + assert_eq!( + result.skipped[0].1, + "property has no non-null vectors to train on yet" + ); + } else { + let result = result.unwrap(); + assert!( + result.staged.is_some(), + "Lance must fold with its reduced partition count" + ); + assert!(result.skipped.is_empty()); + } + assert_eq!( + Dataset::open(&uri).await.unwrap().version().version, + version + ); + } + } + #[tokio::test] async fn schema_retry_refuses_noninitial_or_nonempty_tables_without_mutation() { for case in ["nonempty", "advanced", "unstable", "wrong-format"] { diff --git a/crates/omnigraph/tests/failpoints.rs b/crates/omnigraph/tests/failpoints.rs index d0e1a6822..3ee257973 100644 --- a/crates/omnigraph/tests/failpoints.rs +++ b/crates/omnigraph/tests/failpoints.rs @@ -4475,7 +4475,24 @@ async fn optimize_pre_publish_failure_leaves_no_residue() { .version; { let _failpoint = catalog::OPTIMIZE_POST_PHASE_B_PRE_MANIFEST_COMMIT.fire_always(); - let err = db.optimize().await.unwrap_err(); + let failure = db + .optimize_with_limits_as( + &omnigraph::db::OptimizeLimits { + max_tables: 16, + max_fragments: 1024, + max_rows: 100_000, + max_source_bytes: 64 * 1024 * 1024, + max_files: 4096, + }, + None, + ) + .await + .unwrap_err(); + assert!( + failure.effects_started, + "a lost pre-publication attempt is not a pre-effect refusal" + ); + let err = failure.error; assert!( err.to_string().contains( "injected failpoint triggered: optimize.post_phase_b_pre_manifest_commit" @@ -4695,6 +4712,16 @@ node Embedding { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[serial(optimize)] async fn optimize_partial_table_effect_leaves_no_residue() { + assert_optimize_partial_table_effect_leaves_no_residue(false).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial(optimize)] +async fn bounded_optimize_partial_effect_stops_before_the_next_table() { + assert_optimize_partial_table_effect_leaves_no_residue(true).await; +} + +async fn assert_optimize_partial_table_effect_leaves_no_residue(bounded: bool) { let _scenario = FailScenario::setup(); let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_str().unwrap().to_string(); @@ -4723,8 +4750,43 @@ async fn optimize_partial_table_effect_leaves_no_residue() { .await .version() .version; + let person_versions = detached_versions(&person_uri).await; + let company_versions = detached_versions(&company_uri).await; let _failpoint = catalog::OPTIMIZE_POST_TABLE_EFFECT.fire_once_at(1); - let error = db.optimize().await.unwrap_err(); + let error = if bounded { + let failure = db + .optimize_with_limits_as( + &omnigraph::db::OptimizeLimits { + max_tables: 16, + max_fragments: 1024, + max_rows: 100_000, + max_source_bytes: 64 * 1024 * 1024, + max_files: 4096, + }, + None, + ) + .await + .unwrap_err(); + assert!(failure.effects_started); + assert_eq!( + detached_versions(&person_uri).await, + person_versions, + "the bounded attempt must not start the next table after an error" + ); + assert!( + detached_versions(&company_uri).await.len() > company_versions.len(), + "the first sorted table must have reached a detached effect" + ); + failure.error + } else { + db.optimize().await.unwrap_err() + }; + assert!( + error + .to_string() + .contains("injected failpoint triggered: optimize.post_table_effect"), + "unexpected error: {error}" + ); assert!( !matches!(error, OmniError::RecoveryRequired { .. }), "a partial batch before publication needs no recovery: {error}" diff --git a/crates/omnigraph/tests/integration/maintenance.rs b/crates/omnigraph/tests/integration/maintenance.rs index 15afbc9e8..a49f10cca 100644 --- a/crates/omnigraph/tests/integration/maintenance.rs +++ b/crates/omnigraph/tests/integration/maintenance.rs @@ -16,8 +16,8 @@ use lance::dataset::optimize::{CompactionOptions, compact_files}; use lance_core::datatypes::BlobHandling; use omnigraph::IndexCoverage; use omnigraph::db::{ - CleanupPolicyOptions, MergeOutcome, Omnigraph, ReadTarget, RepairAction, RepairClassification, - RepairOptions, SnapshotId, + CleanupPolicyOptions, MergeOutcome, Omnigraph, OptimizeLimits, OptimizeSettlementOutcome, + ReadTarget, RepairAction, RepairClassification, RepairOptions, SnapshotId, }; use omnigraph::error::{BranchStartRefusal, OmniError}; use omnigraph::loader::LoadMode; @@ -32,6 +32,16 @@ use crate::helpers::{ mixed_params, mutate_main, snapshot_main, }; +fn optimize_limits() -> OptimizeLimits { + OptimizeLimits { + max_tables: 16, + max_fragments: 1024, + max_rows: 100_000, + max_source_bytes: 64 * 1024 * 1024, + max_files: 4096, + } +} + /// Filesystem URI of the live main-branch incarnation of a node table. /// Physical paths are identity-derived, so test fixtures must resolve the /// authoritative manifest registration instead of reconstructing a path from @@ -154,7 +164,23 @@ async fn optimize_after_load_then_again_is_idempotent() { let db = init_and_load(&dir).await; // First pass may compact (load wrote real fragments). - let _first = db.optimize().await.unwrap(); + let before = db.list_commits(None).await.unwrap()[0] + .graph_commit_id + .clone(); + let first = db + .optimize_with_limits_as(&optimize_limits(), None) + .await + .unwrap(); + assert_eq!(first.before_commit, before); + assert_eq!( + first.after_commit, + db.list_commits(None).await.unwrap()[0].graph_commit_id + ); + assert_eq!(first.source_tables, 5); + assert!(first.source_fragments > 0); + assert!(first.source_rows > 0); + assert!(first.source_bytes > 0); + assert!(first.source_files > 0); let commits_before = db.list_commits(None).await.unwrap(); let head_before = commits_before @@ -165,8 +191,13 @@ async fn optimize_after_load_then_again_is_idempotent() { // Second pass should be a no-op: already-compacted graph produces no // fragments_removed / fragments_added. - let second = db.optimize().await.unwrap(); - for s in &second { + let second = db + .optimize_with_limits_as(&optimize_limits(), None) + .await + .unwrap(); + assert_eq!(second.before_commit, head_before); + assert_eq!(second.after_commit, head_before); + for s in &second.tables { assert_eq!( s.fragments_removed, 0, "{} re-optimize should be no-op", @@ -200,6 +231,427 @@ async fn optimize_after_load_then_again_is_idempotent() { ); } +#[tokio::test] +async fn bounded_optimize_refuses_complete_inventory_and_failed_binding_before_effects() { + let dir = tempfile::tempdir().unwrap(); + let db = init_and_load(&dir).await; + add_person_fragments(&db).await; + let snapshot = snapshot_main(&db).await.unwrap(); + let before = db.list_commits(None).await.unwrap()[0] + .graph_commit_id + .clone(); + let person = node_table_uri(&db, "Person").await; + let versions = detached_versions(&person).await; + let mut cases = Vec::new(); + for field in 0..5 { + let mut limits = optimize_limits(); + match field { + 0 => limits.max_tables = 1, + 1 => limits.max_fragments = 1, + 2 => limits.max_rows = 1, + 3 => limits.max_source_bytes = 1, + 4 => limits.max_files = 1, + _ => unreachable!(), + } + cases.push(limits.clone()); + match field { + 0 => limits.max_tables = 0, + 1 => limits.max_fragments = 0, + 2 => limits.max_rows = 0, + 3 => limits.max_source_bytes = 0, + 4 => limits.max_files = 0, + _ => unreachable!(), + } + cases.push(limits); + } + for limits in cases { + let failure = db.optimize_with_limits_as(&limits, None).await.unwrap_err(); + assert!(!failure.effects_started, "{limits:?}: {failure:?}"); + assert_eq!(detached_versions(&person).await, versions); + assert_eq!( + db.list_commits(None).await.unwrap()[0].graph_commit_id, + before + ); + } + let (input_tx, input_rx) = tokio::sync::oneshot::channel(); + let failure = db + .optimize_with_limits_admitted_as(&optimize_limits(), None, |input| async move { + assert_eq!(input.source_tables, 5); + input_tx.send(input).unwrap(); + Err(OmniError::manifest("test invocation binding refused")) + }) + .await + .unwrap_err(); + assert!(!failure.effects_started); + assert!( + failure + .error + .to_string() + .contains("test invocation binding refused") + ); + assert_eq!(detached_versions(&person).await, versions); + let after = snapshot_main(&db).await.unwrap(); + for key in ["node:Person", "node:Company", "edge:WorksAt", "edge:Knows"] { + if let Some(entry) = snapshot.dataset(key) { + assert_same_dataset_entry(entry, after.dataset(key).unwrap()); + } + } + + // Exercise the far end of the inventory, not only the first catalog file: + // every ceiling is inclusive, and even the final source reference must be + // admitted before an earlier table can produce detached effects. + let input = input_rx.await.unwrap(); + let exact = OptimizeLimits { + max_tables: input.source_tables, + max_fragments: input.source_fragments, + max_rows: input.source_rows, + max_source_bytes: input.source_bytes, + max_files: input.source_files, + }; + for field in 0..5 { + let mut limits = exact.clone(); + match field { + 0 => limits.max_tables -= 1, + 1 => limits.max_fragments -= 1, + 2 => limits.max_rows -= 1, + 3 => limits.max_source_bytes -= 1, + 4 => limits.max_files -= 1, + _ => unreachable!(), + } + let failure = db.optimize_with_limits_as(&limits, None).await.unwrap_err(); + assert!(!failure.effects_started, "{limits:?}: {failure:?}"); + assert_eq!(detached_versions(&person).await, versions); + assert_eq!( + db.list_commits(None).await.unwrap()[0].graph_commit_id, + before + ); + } + let (publication_tx, publication_rx) = tokio::sync::oneshot::channel(); + let report = db + .optimize_with_limits_recorded_as( + &exact, + None, + |_| async { Ok(()) }, + |publication| async move { + publication_tx.send(publication).unwrap(); + Ok(()) + }, + ) + .await + .unwrap(); + assert_eq!(report.before_commit, input.before_commit); + assert_eq!(report.source_tables, input.source_tables); + assert_eq!(report.source_fragments, input.source_fragments); + assert_eq!(report.source_rows, input.source_rows); + assert_eq!(report.source_bytes, input.source_bytes); + assert_eq!(report.source_files, input.source_files); + assert!(serde_json::to_vec(&report).unwrap().len() as u64 <= input.report_max_bytes); + let publication = publication_rx.await.unwrap(); + assert_eq!( + publication.before_manifest_version, + snapshot.graph_manifest_version() + ); + assert_eq!(publication.report.before_commit, before); + assert_eq!(publication.report.after_commit, before); + assert_eq!(publication.actor, None); + assert_eq!(input.schema_contract, db.schema_contract_digest()); + assert_eq!(publication.schema_contract, input.schema_contract); + assert_eq!( + publication.report.tables, + report + .tables + .iter() + .filter(|table| table.type_key != "__manifest") + .cloned() + .collect::>() + ); + assert!(report.after_commit.contains(&publication.intent_nonce)); + assert!( + serde_json::to_vec(&publication.report).unwrap().len() as u64 <= input.report_max_bytes + ); +} + +#[tokio::test] +async fn bounded_optimize_includes_manifest_records_on_an_empty_graph() { + let dir = tempfile::tempdir().unwrap(); + let db = Omnigraph::init(dir.path().to_str().unwrap(), TEST_SCHEMA) + .await + .unwrap(); + let before = db.list_commits(None).await.unwrap()[0] + .graph_commit_id + .clone(); + let mut limits = optimize_limits(); + // Data tables are empty. This must still refuse: the accepted __manifest + // records are physical inputs to publication and config maintenance. + limits.max_rows = 1; + let failure = db.optimize_with_limits_as(&limits, None).await.unwrap_err(); + assert!(!failure.effects_started); + assert!( + matches!(failure.error, OmniError::ResourceLimitExceeded {resource, ..} + if resource == "optimize_source_rows") + ); + let report = db + .optimize_with_limits_as(&optimize_limits(), None) + .await + .unwrap(); + assert_eq!(report.source_tables, 5); + assert!(report.source_rows > 1); + assert!(report.source_fragments > 0); + assert_eq!(report.before_commit, before); + assert_eq!(report.after_commit, before); +} + +#[tokio::test] +async fn bounded_optimize_recovery_settles_exact_publication_without_replaying_work() { + let dir = tempfile::tempdir().unwrap(); + let db = init_and_load(&dir).await; + add_person_fragments(&db).await; + let source = snapshot_main(&db).await.unwrap(); + let person = node_table_uri(&db, "Person").await; + let (publication_tx, publication_rx) = tokio::sync::oneshot::channel(); + let failure = db + .optimize_with_limits_recorded_as( + &optimize_limits(), + None, + |_| async { Ok(()) }, + |publication| async move { + publication_tx.send(publication).unwrap(); + Err(OmniError::manifest( + "stop after detached work, before publication", + )) + }, + ) + .await + .unwrap_err(); + assert!(failure.effects_started); + let original = publication_rx.await.unwrap(); + let detached = detached_versions(&person).await; + assert_eq!( + snapshot_main(&db).await.unwrap().graph_manifest_version(), + source.graph_manifest_version() + ); + + let settlement = db + .prepare_optimize_settlement_as(&original, None) + .await + .unwrap(); + assert_eq!( + db.settle_optimize_as(&original, &settlement, None) + .await + .unwrap(), + OptimizeSettlementOutcome::Unpublished, + ); + let settled = snapshot_main(&db).await.unwrap(); + assert_eq!( + settled.graph_manifest_version(), + source.graph_manifest_version() + 1 + ); + for entry in source.datasets() { + assert_same_dataset_entry(entry, settled.dataset(&entry.type_key).unwrap()); + } + assert_eq!( + detached_versions(&person).await, + detached, + "settlement must not replay table work" + ); + assert_eq!( + db.settle_optimize_as(&original, &settlement, None) + .await + .unwrap(), + OptimizeSettlementOutcome::Unpublished, + ); + assert_eq!( + snapshot_main(&db).await.unwrap().graph_manifest_version(), + settled.graph_manifest_version() + ); + + let (publication_tx, publication_rx) = tokio::sync::oneshot::channel(); + let report = db + .optimize_with_limits_recorded_as( + &optimize_limits(), + None, + |_| async { Ok(()) }, + |publication| async move { + publication_tx.send(publication).unwrap(); + Ok(()) + }, + ) + .await + .unwrap(); + let original = publication_rx.await.unwrap(); + let settlement = db + .prepare_optimize_settlement_as(&original, None) + .await + .unwrap(); + let mut expected = report.clone(); + expected + .tables + .retain(|table| table.type_key != "__manifest"); + let expected = OptimizeSettlementOutcome::Published { report: expected }; + assert_eq!( + db.settle_optimize_as(&original, &settlement, None) + .await + .unwrap(), + expected + ); + mutate_main( + &db, + MUTATION_QUERIES, + "insert_person", + &mixed_params(&[("$name", "Later")], &[("$age", 30)]), + ) + .await + .unwrap(); + let later = snapshot_main(&db).await.unwrap(); + let detached = detached_versions(&person).await; + assert_eq!( + db.settle_optimize_as(&original, &settlement, None) + .await + .unwrap(), + expected + ); + + let mut tampered = original.clone(); + tampered.intent_nonce = ulid::Ulid::new().to_string(); + assert!( + db.settle_optimize_as(&tampered, &settlement, None) + .await + .is_err() + ); + tampered.intent_nonce = "not-a-native-nonce".to_string(); + assert!( + db.prepare_optimize_settlement_as(&tampered, None) + .await + .is_err() + ); + for field in 0..2 { + let mut unknown = original.clone(); + if field == 0 { + unknown.before_manifest_version = u64::MAX - 1; + } else { + unknown.schema_contract.source_hash = "0".repeat(64); + } + let token = db + .prepare_optimize_settlement_as(&unknown, None) + .await + .unwrap(); + assert_eq!( + db.settle_optimize_as(&unknown, &token, None).await.unwrap(), + OptimizeSettlementOutcome::Unknown + ); + } + assert_eq!( + snapshot_main(&db).await.unwrap().graph_manifest_version(), + later.graph_manifest_version() + ); + assert_eq!(detached_versions(&person).await, detached); +} + +#[tokio::test] +async fn bounded_optimize_does_not_acknowledge_failed_artifact_publication() { + let dir = tempfile::tempdir().unwrap(); + let db = init_and_load(&dir).await; + mutate_main( + &db, + MUTATION_QUERIES, + "add_friend", + &mixed_params(&[("$from", "Charlie"), ("$to", "Diana")], &[]), + ) + .await + .unwrap(); + let before = db.list_commits(None).await.unwrap()[0] + .graph_commit_id + .clone(); + // A directory at the artifact's exact object path lets graph work succeed + // but makes the real local adapter's final atomic replacement fail. + std::fs::create_dir_all(dir.path().join("__graph_index/csr-current.bin")).unwrap(); + let failure = db + .optimize_with_limits_as(&optimize_limits(), None) + .await + .unwrap_err(); + assert!( + failure.effects_started, + "artifact failure follows a possible publication" + ); + assert_ne!( + db.list_commits(None).await.unwrap()[0].graph_commit_id, + before, + "the failure must not be misreported as a pre-effect refusal or success" + ); +} + +#[tokio::test] +async fn bounded_optimize_binds_exact_input_under_writer_gates_and_keeps_reads_available() { + let dir = tempfile::tempdir().unwrap(); + let db = init_and_load(&dir).await; + add_person_fragments(&db).await; + let before = db.list_commits(None).await.unwrap()[0] + .graph_commit_id + .clone(); + let (input_tx, input_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let optimize_db = db.clone(); + let optimize = tokio::spawn(async move { + optimize_db + .optimize_with_limits_admitted_as( + &optimize_limits(), + Some("maintenance-actor"), + |input| async move { + input_tx.send(input).unwrap(); + release_rx.await.unwrap(); + Ok(()) + }, + ) + .await + }); + let input = input_rx.await.unwrap(); + assert_eq!(input.before_commit, before); + // This read completes while maintenance holds every writer gate. + let snapshot = tokio::time::timeout(Duration::from_secs(10), snapshot_main(&db)) + .await + .unwrap() + .unwrap(); + assert!(snapshot.dataset("node:Person").is_some()); + let write_db = db.clone(); + let mut write = tokio::spawn(async move { + mutate_main( + &write_db, + MUTATION_QUERIES, + "insert_person", + &mixed_params(&[("$name", "After")], &[("$age", 45)]), + ) + .await + }); + assert!( + tokio::time::timeout(Duration::from_millis(50), &mut write) + .await + .is_err() + ); + release_tx.send(()).unwrap(); + let report = optimize.await.unwrap().unwrap(); + write.await.unwrap().unwrap(); + assert_eq!(report.before_commit, input.before_commit); + assert_eq!(report.source_tables, input.source_tables); + assert_eq!(report.source_fragments, input.source_fragments); + assert_eq!(report.source_rows, input.source_rows); + assert_eq!(report.source_bytes, input.source_bytes); + assert_eq!(report.source_files, input.source_files); + let commits = db.list_commits(None).await.unwrap(); + let own_commit = commits + .iter() + .find(|commit| commit.graph_commit_id == report.after_commit) + .unwrap(); + assert_eq!( + own_commit.parent_commit_id.as_deref(), + Some(before.as_str()) + ); + assert_eq!(own_commit.actor_id.as_deref(), Some("maintenance-actor")); + assert_ne!( + commits[0].graph_commit_id, report.after_commit, + "report must retain its own publication, not a later writer's head" + ); +} + /// Optimize preserves main's cold registry files and keeps subsequent publishes readable. #[tokio::test] async fn optimize_preserves_manifest_registry_fragments() { @@ -462,7 +914,17 @@ node Doc { ); } - let stats = db.optimize().await.unwrap(); + let (input_tx, input_rx) = tokio::sync::oneshot::channel(); + let report = db + .optimize_with_limits_admitted_as(&optimize_limits(), None, |input| async move { + input_tx.send(input).unwrap(); + Ok(()) + }) + .await + .unwrap(); + let input = input_rx.await.unwrap(); + assert!(serde_json::to_vec(&report).unwrap().len() as u64 <= input.report_max_bytes); + let stats = report.tables; // Postcondition: optimize_indices folded the scalar tail in, but preserves // the existing FTS artifact instead of merging uncertified postings. @@ -644,9 +1106,19 @@ node Tag {\n slug: String @key\n}\n"; .clone(); let probes = omnigraph::instrumentation::MergeWriteProbes::default(); - let stats = omnigraph::instrumentation::with_merge_write_probes(probes.clone(), db.optimize()) - .await - .expect("optimize must not crash on a graph with a Blob table"); + let report = omnigraph::instrumentation::with_merge_write_probes( + probes.clone(), + db.optimize_with_limits_as(&optimize_limits(), None), + ) + .await + .expect("optimize must not crash on a graph with a Blob table"); + assert_eq!(report.before_commit, head_before); + assert_eq!(report.source_tables, 3); + assert!( + report.source_rows >= 9, + "data rows plus accepted manifest records" + ); + let stats = report.tables; assert_eq!( ( probes.compaction_blob_batch_calls(), @@ -1325,9 +1797,32 @@ async fn optimize_then_cleanup_erases_dropped_property_values() { let commits_before = db.list_commits(None).await.unwrap().len(); let probes = omnigraph::instrumentation::MergeWriteProbes::default(); - let stats = omnigraph::instrumentation::with_merge_write_probes(probes.clone(), db.optimize()) + let limits = optimize_limits(); + let mut rejected = limits.clone(); + rejected.max_source_bytes = MIB as u64 - 1; + let failure = db + .optimize_with_limits_as(&rejected, None) .await - .unwrap(); + .unwrap_err(); + assert!( + !failure.effects_started, + "Blob payload limit must precede dropped-column rewrite" + ); + assert_eq!( + pinned_file_field_ids(&db, "node:Doc").await, + files_after_drop + ); + let report = omnigraph::instrumentation::with_merge_write_probes( + probes.clone(), + db.optimize_with_limits_as(&limits, None), + ) + .await + .unwrap(); + assert!( + report.source_bytes >= MIB as u64, + "kept managed Blob bytes are admitted" + ); + let stats = report.tables; let stat = |key: &str| stats.iter().find(|stat| stat.type_key == key).unwrap(); let doc = stat("node:Doc"); assert!( diff --git a/crates/omnigraph/tests/policy_engine_chassis.rs b/crates/omnigraph/tests/policy_engine_chassis.rs index b0b806cc4..9c14a711d 100644 --- a/crates/omnigraph/tests/policy_engine_chassis.rs +++ b/crates/omnigraph/tests/policy_engine_chassis.rs @@ -54,7 +54,7 @@ rules: - id: writers-branches-schema allow: actors: { group: writers } - actions: [schema_apply, branch_create, branch_delete, branch_merge] + actions: [schema_apply, optimize, branch_create, branch_delete, branch_merge] target_branch_scope: any "#; @@ -88,6 +88,26 @@ async fn init_with_policy(dir: &tempfile::TempDir) -> (Session, Arc::`; the ledger ID and +nonce are canonical ULIDs and the sequence is a canonical positive integer. +Maintenance has its own increasing sequence. Read-only status exposes the +next sequence but reserves nothing. Acceptance consumes one sequence and +persists the exact graph, kind, options, budget and authenticated actor before +any effect. It returns a single-use in-process execution token. No caller can +construct an accepted token from the serialized journal. + +After the engine completes source preflight under its graph gates, it +conditionally binds the exact input commit and source inventory to that +invocation before any physical rewrite. Success must match this binding. +A failed binding write retains uncertainty and never permits execution. +The invocation also records its applied configuration revision and schema +contract; these identify admission, while the immutable input commit binds +the actual graph schema and table pins used by optimization. + +Only one outstanding invocation is allowed per cluster. A duplicate identity +and matching input/actor returns the recorded status without execution. +Different input or actor refuses. A different nonce at a retained sequence +refuses. A consumed sequence whose result was evicted reports expired, +unknown acceptance/outcome; it never becomes executable again. Future +sequences and another ledger are refused. + +Completion conditionally records the exact native report or a proven +pre-effect refusal. A refusal consumes its ID and releases the slot; an +uncertain failure keeps it outstanding. Keep at most 32 +terminal reports, at most 1 MiB each and 4 MiB combined. The complete journal +is limited to 8 MiB. Eviction only removes terminal result payloads; the +monotonic sequence retains the no-replay boundary. Outstanding work cannot +be evicted. A result larger than its retention bound leaves uncertainty rather +than publishing a truncated success. + +Before physical effects, the engine derives a conservative report bound from +the exact accepted table and index strings, all supported pending-index +outcomes and maximal numeric widths. Cluster admission includes the complete +invocation/result wrapper, prepared witness and interrupted-result shapes in +the reservation. An envelope that cannot fit refuses before physical work. +The final exact-size check remains a defensive guard. No report is truncated, +and no input/output size ratio substitutes for this proof. + +A crash or storage error between invocation acceptance, graph publication and +result recording leaves an outstanding operation. Lookup labels it unresolved +when no local owner exists. Neither a new process nor another request replays +it or infers success from equal data or a changed head. It preserves its evidence +and refuses new maintenance, deployment and writable process admission. Even +after explicit predecessor exclusion, a successor cannot start ordinary +serving or direct graph operations while the journal is unresolved. The +original process can expose its exact receipt while its admission remains +available. Explicit recovery takes a restricted admission for the original +operation after the administrator establishes predecessor and accepted-I/O +exclusion and handles the exact old lock. It cannot serve requests, deploy +configuration or rerun optimization. + +Before publication, retain the exact native publication witness and prepared +report. Recovery compares that witness against native graph authority. No +prepared witness means graph publication was never attempted. A prepared +candidate's absence alone does not exclude a delayed storage write: use the +existing native no-effect settlement publication to resolve that race, then +validate the exact winner. Missing, conflicting or unclassifiable evidence +leaves the original invocation unresolved. A recovered receipt explicitly +reports interrupted unpublished or interrupted published work; publication +does not claim subsequent derived-artifact work completed. Detached output +remains subject to ordinary native cleanup safety. Recovery never deletes it. + +Current `optimize` permission authorizes status for the requested graph. The +exact authenticated initiator may retrieve its retained individual receipt +after permission revocation, without gaining graph data or other operations. +Missing or expired ownership evidence cannot grant this exception. + +## Bounds and enablement + +Served optimization is explicitly enabled at boot; it is off by default. +The optional `OMNIGRAPH_MAINTENANCE_CONFIG` names a local YAML file (at most +4 KiB) containing positive `max_tables`, `max_fragments`, `max_rows`, +`max_source_bytes` and `max_files`. Embedders may provide the same limits +through the server builder. There are no implicit defaults that enable work. +The server chooses finite ceilings for source tables, fragments, rows, bytes, +file references and dataset concurrency. Requests may lower those ceilings, never raise them. +The first implementation uses one dataset at a time. Limits are validated +before durable acceptance and the complete accepted work is checked before +the first physical effect, including dropped-column rewrite tasks. + +Source accounting includes `__manifest` and all pinned table manifests, data and deletion +files, index-file inventories and live managed Blob lengths. It conservatively +counts whole accepted tables, including dropped-column rewrites; Blob lengths +may count bytes already inside a data file, and each nonempty managed Blob +counts as a file reference even when values share a packed file. External Blob +payloads are not read by compaction. Missing file sizes use exact object HEADs; +unknown index inventories or physical formats, inherited storage bases, system +indexes, overlays and unsupported Blob layouts refuse before effects. Index +accounting inspects native metadata before reader compatibility filtering; +an unsupported index cannot disappear from the accepted inventory. +No historical prefix is listed. +Unknown sizes or unsupported accounting refuse; a per-dataset concurrency +setting alone is not a whole-operation limit. A bounded input is not a claim +of a hard process memory, CPU or storage-rate cap. Live enablement additionally +requires process/platform resource isolation and measured limits. An elapsed +HTTP deadline never interrupts an uncertain storage write. + +This accounting was checked against the pinned Lance 13.0.0 source: +[`CompactionOptions`, planning and task execution](https://docs.rs/crate/lance/13.0.0/source/src/dataset/optimize.rs), +[`Fragment` and `DataFile` metadata](https://docs.rs/crate/lance-table/13.0.0/source/src/format/fragment.rs), +[`IndexMetadata.files`](https://docs.rs/crate/lance-table/13.0.0/source/src/format/index.rs), +the native [`deletion_file_path`](https://docs.rs/crate/lance-table/13.0.0/source/src/io/deletion.rs), +[`Dataset::object_store` and `branch_location`](https://docs.rs/crate/lance/13.0.0/source/src/dataset.rs), +and [`ObjectStore::size`](https://docs.rs/crate/lance-io/13.0.0/source/src/object_store.rs). +Lance's own compaction byte option omits Blob-v2 payloads and can select only +part of the work. It therefore does not substitute for the complete accepted +inventory checked here. These source checks establish the accounting contract, +not a measured process-memory or throughput limit. + +Index reconciliation defers an existing vector index only when it has no +non-null vectors. With positive input, native Lance chooses its supported +training layout. Statistics, storage and index builder errors propagate; they +cannot masquerade as insufficient training data. +New declared vector indexes with no non-null rows retain their existing pending +status. Full-text reconciliation keeps its explicit rebuild boundary. + +No externally scheduled capacity allocation is defined here. A deployment +requiring a fleet-wide budget must keep this capability disabled until native +admission validates that allocation on every entry point. A proxy-only check +is insufficient. Root-bound signed-token and OIDC identity profiles refuse +new optimization at boot and native request admission while this allocation +contract is absent. Disabling execution retains original-ID lookup. +Independent workers additionally require retained immutable +inputs, serialized bounded plans, output-only credentials and native result +validation/publication; none is implied by this endpoint. + +## HTTP and CLI + +Use the shared operation registry and existing HTTP version: + +- `GET /cluster/maintenance?graph=`: authorized read-only status and next ID + components for one graph. +- `POST /cluster/maintenance`: exact operation ID, graph, `optimize` kind and + bounded options; return after durable acceptance or a recorded refusal. +- `GET /cluster/maintenance/{id}?graph=`: exact original result/status + lookup with the original graph, without scanning other graph policies. + +Malformed input, denied policy, disabled capability and exhausted admission +refuse before physical effects. An ambiguous POST response is not a refusal. +Clients retain the original ID and use GET; no automatic POST replay. + +These cluster maintenance routes have no MCP binding in this slice. Their +durable invocation and recovery contract is separate from the non-streaming +graph operations exposed as MCP tools. The shared registry records this +explicit transport boundary. + +Extend the existing `optimize` command and target resolution. Remote execution +waits by default; `--no-wait` returns the known operation ID. +`--operation-id ` observes that original operation without submitting it. +`--timeout` bounds observation only. Direct storage retains its existing +command and output; remote refusal never falls back to direct storage. +For administrator recovery, `optimize --operation-id --writers-stopped` +requires an explicit direct-root target and graph. The attestation means the +prior process and accepted graph/control I/O are excluded, as for deployment +recovery; it never overrides a held lock. This operation reconciles the +original receipt and never executes a new optimization. + +## Evidence required + +Extend existing maintenance, policy, cluster, server and CLI test owners. +Cover denied and allowed actors, complete bounds before effects, concurrent +reads and write contention, identical/mismatched duplicates, response loss, +result eviction, accepted-operation interruption, shutdown and restart lookup. +Compare logical rows, schema, Blob values, index behavior, retained history and +live branches before and after. Test failures before and after publication +without treating a lost response as permission to execute again. + +Local tests establish the candidate's contract. They do not qualify cloud +storage settlement, fleet admission or production resource ceilings. diff --git a/docs/rfcs/README.md b/docs/rfcs/README.md index 5b8afb7a7..60c56421b 100644 --- a/docs/rfcs/README.md +++ b/docs/rfcs/README.md @@ -254,3 +254,4 @@ then dated RFCs by date. | [2026-10-19](2026-10-19-http-api-conventions.md) | HTTP API conventions | maintainer | accepted | complete | | [2026-10-19](2026-10-19-storage-outcomes-and-restartable-ownership.md) | Storage outcomes per operation and restartable ownership | maintainer | accepted | in-progress | | [2026-10-20](2026-10-20-operator-connections-and-credential-providers.md) | Operator profiles and credential providers | maintainer | draft | in-progress | +| [2026-10-25](2026-10-25-served-optimization.md) | Served optimization | maintainer | draft | in-progress | diff --git a/docs/user/cli/reference.md b/docs/user/cli/reference.md index 00d806651..218754515 100644 --- a/docs/user/cli/reference.md +++ b/docs/user/cli/reference.md @@ -55,7 +55,7 @@ operations on nonlocal storage. The retired `--quiet` cannot hide the destinatio | `schema upgrade-system-columns` | Respell a graph's system columns in place; needs a graph that normal open accepts and keeps its storage format | direct | | `lint` | Validate `.gq` source | local schema or direct graph | | `upgrade` | Convert a v8, v9 or v13 graph's storage to the format this binary serves, offline; `--check` writes nothing | direct standalone | -| `optimize` | Compact data and reconcile declared indexes | direct | +| `optimize` | Compact data and reconcile declared indexes; served work retains an operation ID | direct, server | | `rebuild-full-text-indexes` | Replace full-text indexes on one branch | direct | | `repair` | Report each table's Lance history against its registration (`no_drift` or `foreign_drift`) | direct | | `cleanup` | Delete table versions that no retained graph commit pins, under an explicit retention policy ([Maintenance](../operations/maintenance.md#cleanup)) | direct | diff --git a/docs/user/cli/targets.md b/docs/user/cli/targets.md index 66ab69bea..b97be7a4f 100644 --- a/docs/user/cli/targets.md +++ b/docs/user/cli/targets.md @@ -299,9 +299,9 @@ Version 1 does not support cloud-direct targets, catalogue targets, Unix sockets mTLS, tunnels or arbitrary private identity-provider transport. Server storage and deployment retain separate contracts. This rewrite does not qualify BYOC private routes or workload federation. Managed and standalone targets share -served graph operations subject to capabilities and policy; maintenance, schema -changes and direct-root administration keep their storage restrictions. Remote -failures never trigger direct storage fallback. +served graph operations subject to capabilities and policy. Bounded `optimize` +also supports enabled standalone servers; other maintenance, schema changes +and direct-root administration keep their storage restrictions. Remote failures never trigger direct storage fallback. The old standalone `--target og_catalog` shorthand is not a built-in target; catalogue reads through a registered server still use `--graph og_catalog`. diff --git a/docs/user/operations/maintenance.md b/docs/user/operations/maintenance.md index 7bf20669a..23a891442 100644 --- a/docs/user/operations/maintenance.md +++ b/docs/user/operations/maintenance.md @@ -1,91 +1,127 @@ # Maintenance -OmniGraph provides four direct-storage maintenance commands: +Use `optimize` for compaction and index reconciliation, +`rebuild-full-text-indexes` for one branch's full-text indexes, `repair` to +diagnose foreign storage drift, and `cleanup` to remove unretained versions +and unused table forks left by branches created before storage format v11. -- `optimize` compacts data and reconciles declared indexes. -- `rebuild-full-text-indexes` replaces full-text indexes on one branch. -- `repair` diagnoses foreign storage drift without adopting it. -- `cleanup` permanently removes unretained table versions and unused table - forks left by branches created before storage format v11. - -They do not run through the HTTP server. Address a standalone graph directly, -or select a graph from a cluster root: +`optimize` also supports enabled cluster servers; other commands require direct +storage. Address a standalone graph or select a graph from a cluster root: ```bash omnigraph optimize --target ./graph.omni omnigraph optimize --target ./cluster-root --graph knowledge ``` -Follow the command-specific concurrency requirements below. Operator version 1 -accepts local-file direct targets only; cloud-direct maintenance is not exposed -by this CLI. Engine/server cloud support does not change that client boundary. -See [operator configuration](../cli/targets.md#transport-and-capability-boundaries). +Direct targets support local files only. Follow the concurrency rules below and +[operator configuration](../cli/targets.md#transport-and-capability-boundaries). ## Optimize +Select a graph explicitly or configure the server target's `default_graph`: + ```bash -omnigraph optimize --target ./graph.omni -omnigraph optimize --target ./graph.omni --json +omnigraph optimize --target production --graph knowledge +omnigraph optimize --target production --graph knowledge --no-wait --json +omnigraph optimize --target production --graph knowledge --operation-id '' +``` + +Both modes enforce installed `optimize` policy on `main`. Served work uses the +credential's actor and existing storage-owning server; it refuses `--as` and +never opens storage from the CLI. Direct work selects `--as`, then the operator +configuration's actor. Managed targets do not support optimize yet. + +The CLI flushes the operation ID before submitting once. It waits 300 seconds +by default; `--timeout` accepts 1–3600 seconds and exits 5 without cancelling +work. `--no-wait` returns after acceptance; `--operation-id` only observes and +never submits. Lost responses, unresolved work and expired receipts never +permit replay: retain the ID and inspect the same target and graph. JSON keeps +the native receipt, before/after commits and per-table results. Direct storage +accepts an original ID only with explicit recovery; both modes support +`--json`. + +### Server enablement + +Served optimization is off by default. Set `OMNIGRAPH_MAINTENANCE_CONFIG` to a +local YAML file (at most 4 KiB), loaded at boot, with all five positive input +ceilings. Example for a small qualification fixture: + +```yaml +max_tables: 100 +max_fragments: 1000 +max_rows: 100000 +max_source_bytes: 134217728 +max_files: 10000 ``` +Embedders can call `AppState::with_maintenance_limits`; requests can lower its +ceilings. All source preflight precedes rewriting; unsupported or unbounded +layouts refuse. One optimization runs per cluster, one table at a time. Input +ceilings do not bound CPU, heap or storage rate: size and isolate the process +before enabling. Fleet-managed admission remains unavailable. +Servers with root-bound signed-token or OIDC identity trust refuse new +optimization, including direct HTTP calls, until resource admission is +supported. Their original-ID receipts remain readable. + +Uncertain effects or a lost result write leave the operation outstanding, +without automatic replay. The complete receipt capacity is reserved from +accepted metadata before physical effects; an oversized result bound refuses +before rewriting. Disabling execution does not hide retained receipts. + +Unresolved maintenance also blocks writable restart. See +[interrupted optimization recovery](storage-failures.md#interrupted-optimization) +for the explicit administrator procedure. + +### Data and concurrency + Optimize rewrites small fragments into fewer larger fragments, rewrites every fragment that still stores a dropped property's values, rebuilds each scalar or vector index whose coverage lags behind appended rows, and builds missing declared indexes that are ready to build. It does not delete old versions or collect unused table forks. Use `cleanup` for storage reclamation. -Dropping a property removes it from the schema without rewriting its table -(see [schema changes](../schema/index.md#schema-changes)), so its values stay -in the table's data files. Every optimize rewrites each fragment that still -holds them, whatever its size, copying only the remaining properties; the -dropped values are not read. A `cleanup` then deletes the old files, Blob -values included, once nothing it retains references them: no commit its -retention keeps, no live branch created from a commit before that optimize, -and no tag naming such a commit. Merging a branch does not release its files; -deleting it does. A table holding no dropped values and nothing else to -compact is left alone. - -Each table's work is staged as detached Lance versions of the table's current -pin and published in one graph commit, like any other write. A run -that fails before that commit leaves the graph unchanged and no recovery -state; after it the published pins are the tables' versions, with nothing -left to finish. Optimize runs beside -live writers: a write that lands on a table while its compaction is staged -fails the run with a read-set conflict, and the next run re-plans from the -new state. An update or delete prepared before an optimize published reports -the same conflict and is retried by its caller. Lance compacts neighbouring -fragments only when the same indexes cover them, so a table whose index -coverage was uneven before a run may coalesce fully only on the next run, -after the rebuilt coverage is in place. - -Compacting a table with Blob properties reads each managed Blob value into -memory to rewrite it. Optimize sizes the read batches of each compaction task -(a group of neighbouring fragments compacted together) from the task's largest -row, summing that row's managed Blob values, so one batch materializes at most -32 MiB of managed Blob payload whatever the fragment sizes. External Blob -references are carried without reading the referenced object and count -nothing. A task holding a row over 16 MiB compacts one row per batch, and a -single row over 32 MiB is materialized whole. This bounds the payload of a -batch, not the process heap: Lance's writer copies each inline payload into -the arrays it prepares while it still holds the batch, so memory use exceeds -the payload. Every task of a Blob table gets this batch size explicitly, 1 to -8192 rows, so it takes precedence over `LANCE_DEFAULT_BATCH_SIZE` there, -including when the variable is smaller. When a fragment being compacted holds -a Blob descriptor the engine's decoder rejects, sizing can refuse the -compaction with a Blob integrity error before that table is rewritten, and -nothing is committed; a table with nothing to compact is not scanned. -Optimize works on up to -`OMNIGRAPH_MAINTENANCE_CONCURRENCY` tables at once (default 8), so budget for -that many Blob tables compacting together. +[Dropping a property](../schema/index.md#schema-changes) leaves its values in +data files. Optimize rewrites every fragment still holding them, regardless +of size, copying only remaining properties without reading dropped values. +`cleanup` deletes old files, including Blobs, only after retained commits, +live branches and tags stop referencing them. Merging a branch does not release +its files; deleting it does. A table without dropped values or compaction work +is left alone. + +Work stages as detached Lance versions and publishes together in at most one +graph commit. Proven failure before publication leaves the visible graph +unchanged, though detached files may remain. Lost responses do not establish +publication. Served receipts can remain unresolved before or after the commit: +observe the original operation ID instead of rerunning optimization. + +Optimize captures authority, then acquires native schema, main-branch and table +gates and revalidates before physical work; changed authority refuses. Mutations +prepared against earlier table versions can conflict after optimize publishes. +Lance compacts neighbours only when the same indexes cover them. Uneven index +coverage can therefore require another run after index coverage is rebuilt. + +Blob compaction materializes managed values. Each task (neighbouring fragments +compacted together) sizes batches from its largest row's summed Blob payload +to target at most 32 MiB per batch. External references are copied without +reading or counting their objects. Rows over 16 MiB run one per batch; a row +over 32 MiB is materialized whole. This bounds payload, not heap: Lance retains +the batch while copying payloads into writer arrays. The explicit batch size, +1–8192 rows, overrides `LANCE_DEFAULT_BATCH_SIZE`, even a smaller value. +Invalid Blob descriptors can refuse sizing before that table is rewritten, +with no graph commit; tables without compaction work are not scanned. +Direct optimize uses up to `OMNIGRAPH_MAINTENANCE_CONCURRENCY` tables at once +(default 8); budget for concurrent Blob tables. Bounded served optimize uses +one table at a time, without a peak-memory guarantee. Optimize also persists the traversal-adjacency artifact (`__graph_index/csr-current.bin`), which cold traversal builds load instead of scanning every edge dataset. The artifact is derived and regenerable: optimize is its only writer, every load is verified and falls back to the in-memory -build, and a failed write is a warning, not an optimize error. The artifact is -rewritten by optimize runs that advance an edge dataset; on a store with no -edge work it is left as-is, and a stale copy costs only the artifact's speedup, -never correctness. +build. Direct optimize warns if its write fails. Bounded served optimize +retains an unresolved operation, even if graph publication already succeeded; +inspect the original ID instead of retrying. Optimize rewrites the artifact +when an edge dataset advances; otherwise it leaves it as-is. A stale copy costs +only the artifact's speedup, never correctness. Existing full-text indexes are preserved rather than incrementally merged. Unindexed rows remain searchable, but a growing uncovered tail can cost more to @@ -94,8 +130,9 @@ analyzer generation. Optimize reports uncovered full-text coverage under `pending_indexes` with this remedy. Deferred coverage alone creates no graph commit or maintenance work. -A vector index whose property has no usable vectors remains pending rather than -failing the run. Run optimize again after loading or generating vectors. +Vector indexes with no non-null vectors remain pending. With positive input, +Lance chooses the supported training layout. Statistics, storage and index-build +failures fail optimization instead of being reported as pending work. ## Rebuild full-text indexes @@ -305,14 +342,9 @@ Exit 0 means every table was visited and that nothing a retained graph commit pins, and no file such a commit references, was removed. It does not mean every table was collected: read the result rows for tables with an `error`. -## Suggested cadence - -- Run `optimize` after large loads or on a regular cadence for write-heavy - graphs. -- Run `repair` when you want to know whether something outside OmniGraph - committed to a table's Lance history. -- Run `cleanup` from an explicit retention policy after backups and rollback - requirements have been reviewed. +Run optimize after large loads, repair to diagnose foreign table commits, and +cleanup only with an explicit retention policy that accounts for backups and +rollback requirements. Moving a graph to another storage format is not maintenance. See [Upgrading](upgrade.md). diff --git a/docs/user/operations/policy.md b/docs/user/operations/policy.md index 2187d6e8d..a92a3e865 100644 --- a/docs/user/operations/policy.md +++ b/docs/user/operations/policy.md @@ -16,6 +16,7 @@ Graph-scoped actions: | `export` | Snapshot export | | `change` | Mutations, loads, and source metadata probes for `explain load csv` | | `schema_apply` | Schema changes | +| `optimize` | Compaction and declared-index reconciliation on target branch `main` | | `branch_create` | Branch creation | | `branch_delete` | Branch deletion | | `branch_merge` | Branch merge | @@ -42,6 +43,11 @@ The reserved graph `admin` action does not grant cluster management. A stored mutation requires both `invoke_query` and `change`. A stored read requires `invoke_query` and `read`. +`optimize` is explicit: `change`, `schema_apply`, `config_manage` and the +reserved `admin` action do not imply it. This check applies to direct engine +and CLI optimization whenever a graph policy is installed, as well as served +optimization. Use `target_branch_scope` to restrict it to protected `main`. + ## Bind a policy ```yaml @@ -210,6 +216,11 @@ authentication allows every graph action and reports `actor_id: null`. The MCP `access` tool returns the same body, and `omnigraph access` prints it for a server target ([what you may do at a target](../cli/targets.md#what-you-may-do-at-a-target)). +This report's closed action vocabulary does not include `optimize`; its absence +does not mean denial. On a server with maintenance enabled, read +`GET /cluster/maintenance?graph=` to check current optimize permission on +`main` without submitting work. + It reports policy actions, not actor views, so a view can still refuse a `change` the policy allows. diff --git a/docs/user/operations/storage-failures.md b/docs/user/operations/storage-failures.md index 78476b48f..aee94976b 100644 --- a/docs/user/operations/storage-failures.md +++ b/docs/user/operations/storage-failures.md @@ -99,6 +99,27 @@ conditional update, so a successor could not replace the lock without racing a release the old owner already sent. Older binaries refuse the leased (version 3) lock record; do not delete it to bypass the version check. +## Interrupted optimization + +An administrator can reconcile the original operation after stopping all prior +writers and establishing accepted graph/control I/O quiescence. Handle any +retained lock through the existing exact-ID `cluster force-unlock` procedure +only after that exclusion; recovery itself never overrides a held lock. + +```bash +omnigraph optimize --target ./cluster-root --graph knowledge \ + --operation-id '' --writers-stopped --as act-operator --json +``` + +This requires an explicit direct-root target and current `optimize` permission. +It reads the original publication witness and native graph evidence and never +reruns compaction. Where necessary, one native no-effect publication excludes +a delayed original commit. The terminal outcome is `interrupted`, with +`publication.status` equal to `unpublished` or `published`; the latter includes +the exact published table report and does not claim later derived work finished. +Missing or conflicting evidence keeps the operation unresolved. Preserve the +printed operation and lock IDs if recovery or result delivery fails. + ## Logs No storage-failure log carries an error's message, which can name storage diff --git a/openapi.json b/openapi.json index 4ff6a2afb..a518fc522 100644 --- a/openapi.json +++ b/openapi.json @@ -981,13 +981,22 @@ ] } }, - "/cluster/plan": { - "post": { + "/cluster/maintenance": { + "get": { "tags": [ "cluster" ], - "operationId": "deployment_plan", + "operationId": "maintenanceStatus", "parameters": [ + { + "name": "graph", + "in": "query", + "description": "Graph to optimize; scope is main", + "required": true, + "schema": { + "type": "string" + } + }, { "name": "Omnigraph-Http-Api", "in": "header", @@ -1010,16 +1019,6 @@ } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/PlanRequest" - } - } - }, - "required": true - }, "responses": { "200": { "description": "", @@ -1037,7 +1036,7 @@ "content": { "application/json": { "schema": { - "type": "object" + "$ref": "#/components/schemas/MaintenanceStatusOutput" } } } @@ -1159,27 +1158,6 @@ } } }, - "408": { - "description": "request_timeout when the request body deadline passes before operation admission", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorOutput" - } - } - } - }, "409": { "description": "", "headers": { @@ -1222,48 +1200,6 @@ } } }, - "413": { - "description": "; payload_too_large when the request body exceeds its route limit before operation admission", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorOutput" - } - } - } - }, - "415": { - "description": "", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorOutput" - } - } - } - }, "429": { "description": "too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", "headers": { @@ -1292,7 +1228,7 @@ } }, "500": { - "description": "Internal failure (code `internal`) while evaluating deployment access or planning", + "description": "", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -1345,16 +1281,12 @@ "bearer_token": [] } ] - } - }, - "/graphs": { - "get": { + }, + "post": { "tags": [ - "management" + "cluster" ], - "summary": "List every graph currently registered with this server.", - "description": "Cedar-gated by the server-level policy via the `graph_list` action\nagainst `Omnigraph::Server::\"root\"`.\n\nOrder: alphabetical by `graph_id` (server-sorted so clients see\ndeterministic output across requests).", - "operationId": "listGraphs", + "operationId": "optimize", "parameters": [ { "name": "Omnigraph-Http-Api", @@ -1378,9 +1310,19 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MaintenanceRequest" + } + } + }, + "required": true + }, "responses": { "200": { - "description": "List of registered graphs", + "description": "", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -1395,13 +1337,34 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/GraphListOutput" + "$ref": "#/components/schemas/MaintenanceOutput" + } + } + } + }, + "202": { + "description": "Durable acceptance; inspect the original identity", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MaintenanceOutput" } } } }, "400": { - "description": "api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "description": "; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -1422,7 +1385,7 @@ } }, "401": { - "description": "Unauthorized; unauthorized when no bearer credential is presented or the server refuses it", + "description": "unauthorized when no bearer credential is presented or the server refuses it", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -1449,7 +1412,7 @@ } }, "403": { - "description": "Forbidden", + "description": "", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -1517,8 +1480,8 @@ } } }, - "412": { - "description": "actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", + "408": { + "description": "request_timeout when the request body deadline passes before operation admission", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -1538,8 +1501,8 @@ } } }, - "500": { - "description": "Internal failure (code `internal`) before a successful response", + "409": { + "description": "", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -1558,47 +1521,9 @@ } } } - } - }, - "security": [ - { - "bearer_token": [] - } - ] - } - }, - "/graphs/discovery": { - "get": { - "tags": [ - "management" - ], - "operationId": "discoverGraphs", - "parameters": [ - { - "name": "Omnigraph-Http-Api", - "in": "header", - "description": "exactly one Omnigraph-Http-Api header with value 0.14 is required", - "required": true, - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - } }, - { - "name": "Omnigraph-If-Actor", - "in": "header", - "description": "The actor the caller expects its credential to resolve to. A mismatch is refused with 412 actor_mismatch before graph lookup or effects; a match changes nothing.", - "required": false, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "Authenticated minimal graph inventory", + "412": { + "description": "actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -1613,13 +1538,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/GraphDiscoveryOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } }, - "400": { - "description": "api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "413": { + "description": "payload_too_large when the request body exceeds its route limit before operation admission", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -1639,8 +1564,8 @@ } } }, - "401": { - "description": "Unauthorized; unauthorized when no bearer credential is presented or the server refuses it", + "415": { + "description": "", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -1650,12 +1575,6 @@ ] }, "description": "Supported OmniGraph HTTP API contract." - }, - "WWW-Authenticate": { - "schema": { - "type": "string" - }, - "description": "Bearer challenge (RFC 6750 §3). It carries error=\"invalid_token\" when a presented bearer credential was refused, and resource_metadata naming the OAuth protected-resource metadata when the server has an OIDC resource identity." } }, "content": { @@ -1666,8 +1585,8 @@ } } }, - "403": { - "description": "Identity credential required", + "429": { + "description": "too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -1677,27 +1596,12 @@ ] }, "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorOutput" - } - } - } - }, - "404": { - "description": "not_found when no route serves the path", - "headers": { - "Omnigraph-Http-Api": { + }, + "Retry-After": { "schema": { - "type": "string", - "enum": [ - "0.14" - ] + "type": "string" }, - "description": "Supported OmniGraph HTTP API contract." + "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." } }, "content": { @@ -1708,15 +1612,9 @@ } } }, - "405": { - "description": "method_not_allowed when the path is served, but not with this method", + "500": { + "description": "Internal failure (code `internal`) or uncertain optimization (code `outcome_unknown`); inspect the original identity without resubmission", "headers": { - "Allow": { - "schema": { - "type": "string" - }, - "description": "The methods the path serves." - }, "Omnigraph-Http-Api": { "schema": { "type": "string", @@ -1735,8 +1633,8 @@ } } }, - "412": { - "description": "actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", + "503": { + "description": "; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -1746,6 +1644,12 @@ ] }, "description": "Supported OmniGraph HTTP API contract." + }, + "Retry-After": { + "schema": { + "type": "string" + }, + "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." } }, "content": { @@ -1764,38 +1668,27 @@ ] } }, - "/graphs/{graph_id}/access": { + "/cluster/maintenance/{operation_id}": { "get": { "tags": [ - "access" + "cluster" ], - "summary": "What the caller's own credential may do on this graph.", - "description": "Evaluates the graph's installed policy for the resolved actor once per\naction, with each action's branch and target branch shaped as the route\nthat performs it shapes them. It needs no graph action itself, reads no\ngraph data, and never checks whether a named branch exists.", - "operationId": "access", + "operationId": "maintenanceLookup", "parameters": [ { - "name": "graph_id", + "name": "operation_id", "in": "path", - "description": "Graph id to route the request to. The `og_` prefix is reserved for system graphs. `og_catalog` is the built-in, storage-free catalogue: it serves only POST /query and GET /schema with their ordinary response bodies. Its query refuses `branch` and `snapshot`; every other route under /graphs/og_catalog returns 405 `method_not_allowed` before effects.", + "description": "Original maintenance identity", "required": true, "schema": { "type": "string" } }, { - "name": "branch", + "name": "graph", "in": "query", - "description": "A branch whose actions to report, by name; it need not exist.", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "target_branch", - "in": "query", - "description": "With `branch`, the branch `branch_create` would create and\n`branch_merge` would merge into, by name.", - "required": false, + "description": "Exact graph of the original operation", + "required": true, "schema": { "type": "string" } @@ -1824,7 +1717,7 @@ ], "responses": { "200": { - "description": "The actions the caller's credential may take", + "description": "", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -1839,13 +1732,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AccessOutput" + "$ref": "#/components/schemas/MaintenanceOutput" } } } }, "400": { - "description": "An unknown or repeated parameter, a target branch without a branch, or a branch name that is not canonical; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "description": "; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -1866,7 +1759,7 @@ } }, "401": { - "description": "Unauthorized; unauthorized when no bearer credential is presented or the server refuses it", + "description": "unauthorized when no bearer credential is presented or the server refuses it", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -1892,8 +1785,29 @@ } } }, + "403": { + "description": "", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, "404": { - "description": "not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "description": "not_found when no route serves the path", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -1940,6 +1854,27 @@ } } }, + "409": { + "description": "", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, "412": { "description": "actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", "headers": { @@ -1989,7 +1924,7 @@ } }, "500": { - "description": "The installed policy could not be evaluated (code `internal`)", + "description": "", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -2010,7 +1945,7 @@ } }, "503": { - "description": "graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "description": "Owner changed during observation; retry this GET; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -2044,126 +1979,13 @@ ] } }, - "/graphs/{graph_id}/blob": { - "get": { + "/cluster/plan": { + "post": { "tags": [ - "blobs" + "cluster" ], - "summary": "Deliver one logical node or edge Blob cell.", - "description": "Managed content is streamed through the bounded transport. External\ndescriptors redirect without target-store I/O. Authorization and target\nresolution share the exact helper used by `/query`.", - "operationId": "blobGet", + "operationId": "deployment_plan", "parameters": [ - { - "name": "graph_id", - "in": "path", - "description": "Graph id to route the request to. The `og_` prefix is reserved for system graphs. `og_catalog` is the built-in, storage-free catalogue: it serves only POST /query and GET /schema with their ordinary response bodies. Its query refuses `branch` and `snapshot`; every other route under /graphs/og_catalog returns 405 `method_not_allowed` before effects.", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "entity", - "in": "query", - "description": "Select a logical node or edge cell.", - "required": true, - "schema": { - "$ref": "#/components/schemas/EntityKind" - } - }, - { - "name": "type", - "in": "query", - "description": "Accepted-schema node or edge type name.", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "id", - "in": "query", - "description": "Logical entity id within the selected type.", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "property", - "in": "query", - "description": "Accepted-schema Blob property name.", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "branch", - "in": "query", - "description": "Branch to read. Mutually exclusive with `snapshot`; defaults to `main`.", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "snapshot", - "in": "query", - "description": "Immutable graph snapshot id. Mutually exclusive with `branch`.", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "If-Match", - "in": "header", - "description": "Strong entity-tag-list precondition, including `*`, evaluated before If-None-Match and Range.", - "required": false, - "schema": { - "type": [ - "string", - "null" - ] - } - }, - { - "name": "Range", - "in": "header", - "description": "One `bytes` range. Malformed, unknown-unit, and multiple ranges are ignored in V1.", - "required": false, - "schema": { - "type": [ - "string", - "null" - ] - } - }, - { - "name": "If-None-Match", - "in": "header", - "description": "Weak entity-tag-list comparison, including `*`, evaluated before Range.", - "required": false, - "schema": { - "type": [ - "string", - "null" - ] - } - }, - { - "name": "If-Range", - "in": "header", - "description": "One strong entity tag. A mismatch causes the complete representation to be served.", - "required": false, - "schema": { - "type": [ - "string", - "null" - ] - } - }, { "name": "Omnigraph-Http-Api", "in": "header", @@ -2186,30 +2008,41 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PlanRequest" + } + } + }, + "required": true + }, "responses": { "200": { - "description": "Complete managed Blob", + "description": "", "headers": { - "Accept-Ranges": { - "schema": { - "type": "string" - }, - "description": "The literal value `bytes` for managed content" - }, - "Content-Length": { + "Omnigraph-Http-Api": { "schema": { - "type": "integer", - "format": "int64", - "minimum": 0 + "type": "string", + "enum": [ + "0.14" + ] }, - "description": "Exact served payload length" - }, - "ETag": { + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { "schema": { - "type": "string" - }, - "description": "Strong validator for the selected managed Blob" - }, + "type": "object" + } + } + } + }, + "400": { + "description": "; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "headers": { "Omnigraph-Http-Api": { "schema": { "type": "string", @@ -2218,50 +2051,20 @@ ] }, "description": "Supported OmniGraph HTTP API contract." - }, - "Omnigraph-Snapshot-Id": { - "schema": { - "type": "string" - }, - "description": "Exact resolved graph snapshot" } }, "content": { - "application/octet-stream": { + "application/json": { "schema": { - "type": "string", - "format": "binary", - "description": "OpenAPI-only marker for an unstructured octet-stream response body." + "$ref": "#/components/schemas/ErrorOutput" } } } }, - "206": { - "description": "One satisfiable managed byte range", + "401": { + "description": "unauthorized when no bearer credential is presented or the server refuses it", "headers": { - "Accept-Ranges": { - "schema": { - "type": "string" - } - }, - "Content-Length": { - "schema": { - "type": "integer", - "format": "int64", - "minimum": 0 - } - }, - "Content-Range": { - "schema": { - "type": "string" - } - }, - "ETag": { - "schema": { - "type": "string" - } - }, - "Omnigraph-Http-Api": { + "Omnigraph-Http-Api": { "schema": { "type": "string", "enum": [ @@ -2270,37 +2073,24 @@ }, "description": "Supported OmniGraph HTTP API contract." }, - "Omnigraph-Snapshot-Id": { + "WWW-Authenticate": { "schema": { "type": "string" - } + }, + "description": "Bearer challenge (RFC 6750 §3). It carries error=\"invalid_token\" when a presented bearer credential was refused, and resource_metadata naming the OAuth protected-resource metadata when the server has an OIDC resource identity." } }, "content": { - "application/octet-stream": { + "application/json": { "schema": { - "type": "string", - "format": "binary", - "description": "OpenAPI-only marker for an unstructured octet-stream response body." + "$ref": "#/components/schemas/ErrorOutput" } } } }, - "302": { - "description": "External Blob descriptor; the server does not dereference it", + "403": { + "description": "", "headers": { - "Cache-Control": { - "schema": { - "type": "string" - }, - "description": "The literal value `no-store`" - }, - "Location": { - "schema": { - "type": "string" - }, - "description": "Exact stored absolute URI" - }, "Omnigraph-Http-Api": { "schema": { "type": "string", @@ -2309,54 +2099,18 @@ ] }, "description": "Supported OmniGraph HTTP API contract." - }, - "Omnigraph-Snapshot-Id": { - "schema": { - "type": "string" - }, - "description": "Exact resolved graph snapshot" } - } - }, - "304": { - "description": "If-None-Match matched the managed Blob validator", - "headers": { - "Accept-Ranges": { - "schema": { - "type": "string" - } - }, - "Content-Length": { - "schema": { - "type": "integer", - "format": "int64", - "minimum": 0 - }, - "description": "Complete managed Blob length, as required for a valid 304 Content-Length" - }, - "ETag": { - "schema": { - "type": "string" - } - }, - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - }, - "Omnigraph-Snapshot-Id": { + }, + "content": { + "application/json": { "schema": { - "type": "string" + "$ref": "#/components/schemas/ErrorOutput" } } } }, - "400": { - "description": "Invalid selector, target, or non-Blob property; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "404": { + "description": "not_found when no route serves the path", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -2376,9 +2130,15 @@ } } }, - "401": { - "description": "Unauthorized; unauthorized when no bearer credential is presented or the server refuses it", + "405": { + "description": "method_not_allowed when the path is served, but not with this method", "headers": { + "Allow": { + "schema": { + "type": "string" + }, + "description": "The methods the path serves." + }, "Omnigraph-Http-Api": { "schema": { "type": "string", @@ -2387,12 +2147,6 @@ ] }, "description": "Supported OmniGraph HTTP API contract." - }, - "WWW-Authenticate": { - "schema": { - "type": "string" - }, - "description": "Bearer challenge (RFC 6750 §3). It carries error=\"invalid_token\" when a presented bearer credential was refused, and resource_metadata naming the OAuth protected-resource metadata when the server has an OIDC resource identity." } }, "content": { @@ -2403,8 +2157,8 @@ } } }, - "403": { - "description": "Forbidden", + "408": { + "description": "request_timeout when the request body deadline passes before operation admission", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -2424,8 +2178,8 @@ } } }, - "404": { - "description": "Unknown entity or null Blob cell; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "409": { + "description": "", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -2445,15 +2199,9 @@ } } }, - "405": { - "description": "method_not_allowed when the path is served, but not with this method", + "412": { + "description": "actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", "headers": { - "Allow": { - "schema": { - "type": "string" - }, - "description": "The methods the path serves." - }, "Omnigraph-Http-Api": { "schema": { "type": "string", @@ -2472,19 +2220,9 @@ } } }, - "412": { - "description": "If-Match did not strongly match the selected managed Blob validator; actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", + "413": { + "description": "; payload_too_large when the request body exceeds its route limit before operation admission", "headers": { - "Accept-Ranges": { - "schema": { - "type": "string" - } - }, - "ETag": { - "schema": { - "type": "string" - } - }, "Omnigraph-Http-Api": { "schema": { "type": "string", @@ -2493,11 +2231,6 @@ ] }, "description": "Supported OmniGraph HTTP API contract." - }, - "Omnigraph-Snapshot-Id": { - "schema": { - "type": "string" - } } }, "content": { @@ -2508,25 +2241,9 @@ } } }, - "416": { - "description": "Requested managed byte range is unsatisfiable", + "415": { + "description": "", "headers": { - "Accept-Ranges": { - "schema": { - "type": "string" - } - }, - "Content-Range": { - "schema": { - "type": "string" - }, - "description": "Unsatisfied range in the form `bytes */N`" - }, - "ETag": { - "schema": { - "type": "string" - } - }, "Omnigraph-Http-Api": { "schema": { "type": "string", @@ -2535,11 +2252,6 @@ ] }, "description": "Supported OmniGraph HTTP API contract." - }, - "Omnigraph-Snapshot-Id": { - "schema": { - "type": "string" - } } }, "content": { @@ -2578,7 +2290,7 @@ } }, "500": { - "description": "Stored Blob integrity failure (code `internal`) or pre-header delivery refusal, including ranged external descriptors that cannot be redirected", + "description": "Internal failure (code `internal`) while evaluating deployment access or planning", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -2631,81 +2343,17 @@ "bearer_token": [] } ] - }, - "put": { + } + }, + "/graphs": { + "get": { "tags": [ - "blobs" + "management" ], - "summary": "Replace one Blob value of an existing node or edge with the request body.", - "description": "The cell's old value is never read; the entity's other cells are carried\nunchanged. Authorization of `change` on the branch runs before the body is\nread. Once admitted the write is owned by the server: a disconnect loses\nonly the response, never cancels or replays the write.", - "operationId": "blobPut", + "summary": "List every graph currently registered with this server.", + "description": "Cedar-gated by the server-level policy via the `graph_list` action\nagainst `Omnigraph::Server::\"root\"`.\n\nOrder: alphabetical by `graph_id` (server-sorted so clients see\ndeterministic output across requests).", + "operationId": "listGraphs", "parameters": [ - { - "name": "graph_id", - "in": "path", - "description": "Graph id to route the request to. The `og_` prefix is reserved for system graphs. `og_catalog` is the built-in, storage-free catalogue: it serves only POST /query and GET /schema with their ordinary response bodies. Its query refuses `branch` and `snapshot`; every other route under /graphs/og_catalog returns 405 `method_not_allowed` before effects.", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "entity", - "in": "query", - "description": "Select a logical node or edge cell.", - "required": true, - "schema": { - "$ref": "#/components/schemas/EntityKind" - } - }, - { - "name": "type", - "in": "query", - "description": "Accepted-schema node or edge type name.", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "id", - "in": "query", - "description": "Logical id of an existing entity within the selected type.", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "property", - "in": "query", - "description": "Accepted-schema Blob property name.", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "branch", - "in": "query", - "description": "Branch to write. Defaults to `main`.", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "If-Match", - "in": "header", - "description": "`*` to require a value in the cell, or a list of strong entity tags one of which must equal the cell's current ETag. Weak tags never match. The write evaluates it against the branch head it applies to.", - "required": false, - "schema": { - "type": [ - "string", - "null" - ] - } - }, { "name": "Omnigraph-Http-Api", "in": "header", @@ -2728,29 +2376,10 @@ } } ], - "requestBody": { - "description": "The raw bytes to store: at most 33554432 bytes (32 MiB), inclusive.", - "content": { - "application/octet-stream": { - "schema": { - "type": "string", - "format": "binary", - "description": "OpenAPI-only marker for an unstructured octet-stream response body." - } - } - }, - "required": true - }, "responses": { "200": { - "description": "The bytes are published in one graph commit", + "description": "List of registered graphs", "headers": { - "ETag": { - "schema": { - "type": "string" - }, - "description": "Strong validator of the stored value; equals `etag` in the body" - }, "Omnigraph-Http-Api": { "schema": { "type": "string", @@ -2764,13 +2393,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/BlobWriteOutput" + "$ref": "#/components/schemas/GraphListOutput" } } } }, "400": { - "description": "Invalid selector, query parameter, If-Match field or target, a non-Blob property, or a carried external reference the graph's policy denies; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "description": "api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -2839,7 +2468,7 @@ } }, "404": { - "description": "Unknown branch, or no entity with this id; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "description": "not_found when no route serves the path", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -2886,8 +2515,8 @@ } } }, - "408": { - "description": "The request body did not arrive before the body deadline; request_timeout when the request body deadline passes before operation admission", + "412": { + "description": "actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -2907,8 +2536,8 @@ } } }, - "409": { - "description": "Write-authority conflict, or the branch's incarnation or accepted schema changed while the write retried", + "500": { + "description": "Internal failure (code `internal`) before a successful response", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -2927,16 +2556,48 @@ } } } + } + }, + "security": [ + { + "bearer_token": [] + } + ] + } + }, + "/graphs/discovery": { + "get": { + "tags": [ + "management" + ], + "operationId": "discoverGraphs", + "parameters": [ + { + "name": "Omnigraph-Http-Api", + "in": "header", + "description": "exactly one Omnigraph-Http-Api header with value 0.14 is required", + "required": true, + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + } }, - "412": { - "description": "If-Match did not hold for the cell; `blob_precondition_failure` names its current validator; actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", + { + "name": "Omnigraph-If-Actor", + "in": "header", + "description": "The actor the caller expects its credential to resolve to. A mismatch is refused with 412 actor_mismatch before graph lookup or effects; a match changes nothing.", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Authenticated minimal graph inventory", "headers": { - "ETag": { - "schema": { - "type": "string" - }, - "description": "The cell's current validator, when it holds a managed value" - }, "Omnigraph-Http-Api": { "schema": { "type": "string", @@ -2950,13 +2611,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/GraphDiscoveryOutput" } } } }, - "413": { - "description": "The body exceeds the server's Blob put limit (the process `write_max_bytes`, at most 32 MiB), or the value with the entity's carried Blob payloads exceeds it; payload_too_large when the request body exceeds its route limit before operation admission", + "400": { + "description": "api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -2976,8 +2637,8 @@ } } }, - "415": { - "description": "Content-Type must be application/octet-stream", + "401": { + "description": "Unauthorized; unauthorized when no bearer credential is presented or the server refuses it", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -2987,6 +2648,12 @@ ] }, "description": "Supported OmniGraph HTTP API contract." + }, + "WWW-Authenticate": { + "schema": { + "type": "string" + }, + "description": "Bearer challenge (RFC 6750 §3). It carries error=\"invalid_token\" when a presented bearer credential was refused, and resource_metadata naming the OAuth protected-resource metadata when the server has an OIDC resource identity." } }, "content": { @@ -2997,8 +2664,8 @@ } } }, - "424": { - "description": "An allowed external Blob source carried from the entity could not be read", + "403": { + "description": "Identity credential required", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -3018,8 +2685,8 @@ } } }, - "429": { - "description": "Per-actor admission cap exceeded; honor `Retry-After` header; too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", + "404": { + "description": "not_found when no route serves the path", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -3029,12 +2696,6 @@ ] }, "description": "Supported OmniGraph HTTP API contract." - }, - "Retry-After": { - "schema": { - "type": "string" - }, - "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." } }, "content": { @@ -3045,9 +2706,15 @@ } } }, - "500": { - "description": "Internal failure (code `internal`), or unsettled write completion (code `outcome_unknown`); follow the returned disposition before another write", + "405": { + "description": "method_not_allowed when the path is served, but not with this method", "headers": { + "Allow": { + "schema": { + "type": "string" + }, + "description": "The methods the path serves." + }, "Omnigraph-Http-Api": { "schema": { "type": "string", @@ -3066,8 +2733,8 @@ } } }, - "503": { - "description": "Write admission is closed or an overlapping durable recovery intent must be resolved before retry; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "412": { + "description": "actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -3077,12 +2744,6 @@ ] }, "description": "Supported OmniGraph HTTP API contract." - }, - "Retry-After": { - "schema": { - "type": "string" - }, - "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." } }, "content": { @@ -3099,14 +2760,16 @@ "bearer_token": [] } ] - }, - "delete": { + } + }, + "/graphs/{graph_id}/access": { + "get": { "tags": [ - "blobs" + "access" ], - "summary": "Set one nullable Blob value of an existing node or edge to null.", - "description": "Clearing a cell that is already null publishes nothing. Like a put, the\nclear is owned by the server once admitted.", - "operationId": "blobClear", + "summary": "What the caller's own credential may do on this graph.", + "description": "Evaluates the graph's installed policy for the resolved actor once per\naction, with each action's branch and target branch shaped as the route\nthat performs it shapes them. It needs no graph action itself, reads no\ngraph data, and never checks whether a named branch exists.", + "operationId": "access", "parameters": [ { "name": "graph_id", @@ -3117,61 +2780,22 @@ "type": "string" } }, - { - "name": "entity", - "in": "query", - "description": "Select a logical node or edge cell.", - "required": true, - "schema": { - "$ref": "#/components/schemas/EntityKind" - } - }, - { - "name": "type", - "in": "query", - "description": "Accepted-schema node or edge type name.", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "id", - "in": "query", - "description": "Logical id of an existing entity within the selected type.", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "property", - "in": "query", - "description": "Accepted-schema Blob property name.", - "required": true, - "schema": { - "type": "string" - } - }, { "name": "branch", "in": "query", - "description": "Branch to write. Defaults to `main`.", + "description": "A branch whose actions to report, by name; it need not exist.", "required": false, "schema": { "type": "string" } }, { - "name": "If-Match", - "in": "header", - "description": "`*` to require a value in the cell, or a list of strong entity tags one of which must equal the cell's current ETag. A null cell satisfies neither form.", + "name": "target_branch", + "in": "query", + "description": "With `branch`, the branch `branch_create` would create and\n`branch_merge` would merge into, by name.", "required": false, "schema": { - "type": [ - "string", - "null" - ] + "type": "string" } }, { @@ -3198,7 +2822,7 @@ ], "responses": { "200": { - "description": "The cell is null. `commit` is the clear's publication, or `null` when the cell already was null and nothing was published", + "description": "The actions the caller's credential may take", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -3213,13 +2837,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/BlobWriteOutput" + "$ref": "#/components/schemas/AccessOutput" } } } }, "400": { - "description": "Invalid selector, query parameter, If-Match field or target, a non-Blob or non-nullable property, or a carried external reference the graph's policy denies; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "description": "An unknown or repeated parameter, a target branch without a branch, or a branch name that is not canonical; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -3266,29 +2890,8 @@ } } }, - "403": { - "description": "Forbidden", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorOutput" - } - } - } - }, "404": { - "description": "Unknown branch, or no entity with this id; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "description": "not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -3335,8 +2938,8 @@ } } }, - "409": { - "description": "Write-authority conflict, or the branch's incarnation or accepted schema changed while the write retried", + "412": { + "description": "actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -3356,15 +2959,9 @@ } } }, - "412": { - "description": "If-Match did not hold for the cell; `blob_precondition_failure` names its current validator; actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", + "429": { + "description": "too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", "headers": { - "ETag": { - "schema": { - "type": "string" - }, - "description": "The cell's current validator, when it holds a managed value" - }, "Omnigraph-Http-Api": { "schema": { "type": "string", @@ -3373,6 +2970,12 @@ ] }, "description": "Supported OmniGraph HTTP API contract." + }, + "Retry-After": { + "schema": { + "type": "string" + }, + "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." } }, "content": { @@ -3383,8 +2986,8 @@ } } }, - "413": { - "description": "The entity's carried Blob payloads exceed the write's payload limit", + "500": { + "description": "The installed policy could not be evaluated (code `internal`)", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -3404,8 +3007,8 @@ } } }, - "424": { - "description": "An allowed external Blob source carried from the entity could not be read", + "503": { + "description": "graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -3415,81 +3018,12 @@ ] }, "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorOutput" - } - } - } - }, - "429": { - "description": "Per-actor admission cap exceeded; honor `Retry-After` header; too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - }, - "Retry-After": { - "schema": { - "type": "string" - }, - "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorOutput" - } - } - } - }, - "500": { - "description": "Internal failure (code `internal`), or unsettled write completion (code `outcome_unknown`); follow the returned disposition before another write", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorOutput" - } - } - } - }, - "503": { - "description": "Write admission is closed or an overlapping durable recovery intent must be resolved before retry; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - }, - "Retry-After": { - "schema": { - "type": "string" - }, - "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." + }, + "Retry-After": { + "schema": { + "type": "string" + }, + "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." } }, "content": { @@ -3506,14 +3040,16 @@ "bearer_token": [] } ] - }, - "head": { + } + }, + "/graphs/{graph_id}/blob": { + "get": { "tags": [ "blobs" ], - "summary": "Return the status and representation headers for one Blob cell.", - "description": "This is a distinct handler rather than Axum's automatic GET-to-HEAD\nfallback. It never calls `BlobReader::read_range`; Range and If-Range are\ndeliberately ignored while If-None-Match is still evaluated.", - "operationId": "blobStat", + "summary": "Deliver one logical node or edge Blob cell.", + "description": "Managed content is streamed through the bounded transport. External\ndescriptors redirect without target-store I/O. Authorization and target\nresolution share the exact helper used by `/query`.", + "operationId": "blobGet", "parameters": [ { "name": "graph_id", @@ -3581,7 +3117,7 @@ { "name": "If-Match", "in": "header", - "description": "Strong entity-tag-list precondition, including `*`, evaluated before If-None-Match.", + "description": "Strong entity-tag-list precondition, including `*`, evaluated before If-None-Match and Range.", "required": false, "schema": { "type": [ @@ -3591,9 +3127,9 @@ } }, { - "name": "If-None-Match", + "name": "Range", "in": "header", - "description": "Weak entity-tag-list comparison, including `*`. Range and If-Range are ignored for HEAD.", + "description": "One `bytes` range. Malformed, unknown-unit, and multiple ranges are ignored in V1.", "required": false, "schema": { "type": [ @@ -3603,9 +3139,9 @@ } }, { - "name": "Range", + "name": "If-None-Match", "in": "header", - "description": "Accepted but ignored for HEAD; metadata always describes the complete selected Blob.", + "description": "Weak entity-tag-list comparison, including `*`, evaluated before Range.", "required": false, "schema": { "type": [ @@ -3617,7 +3153,7 @@ { "name": "If-Range", "in": "header", - "description": "Accepted but ignored for HEAD together with Range.", + "description": "One strong entity tag. A mismatch causes the complete representation to be served.", "required": false, "schema": { "type": [ @@ -3650,13 +3186,13 @@ ], "responses": { "200": { - "description": "Managed Blob metadata with no response body", + "description": "Complete managed Blob", "headers": { "Accept-Ranges": { "schema": { "type": "string" }, - "description": "The literal value `bytes`" + "description": "The literal value `bytes` for managed content" }, "Content-Length": { "schema": { @@ -3664,7 +3200,7 @@ "format": "int64", "minimum": 0 }, - "description": "Complete managed Blob length" + "description": "Exact served payload length" }, "ETag": { "schema": { @@ -3687,6 +3223,65 @@ }, "description": "Exact resolved graph snapshot" } + }, + "content": { + "application/octet-stream": { + "schema": { + "type": "string", + "format": "binary", + "description": "OpenAPI-only marker for an unstructured octet-stream response body." + } + } + } + }, + "206": { + "description": "One satisfiable managed byte range", + "headers": { + "Accept-Ranges": { + "schema": { + "type": "string" + } + }, + "Content-Length": { + "schema": { + "type": "integer", + "format": "int64", + "minimum": 0 + } + }, + "Content-Range": { + "schema": { + "type": "string" + } + }, + "ETag": { + "schema": { + "type": "string" + } + }, + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + }, + "Omnigraph-Snapshot-Id": { + "schema": { + "type": "string" + } + } + }, + "content": { + "application/octet-stream": { + "schema": { + "type": "string", + "format": "binary", + "description": "OpenAPI-only marker for an unstructured octet-stream response body." + } + } } }, "302": { @@ -3759,7 +3354,7 @@ } }, "400": { - "description": "Invalid selector, target, or non-Blob property; HEAD responses have no body; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "description": "Invalid selector, target, or non-Blob property; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -3770,10 +3365,17 @@ }, "description": "Supported OmniGraph HTTP API contract." } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } } }, "401": { - "description": "Unauthorized; HEAD responses have no body; unauthorized when no bearer credential is presented or the server refuses it", + "description": "Unauthorized; unauthorized when no bearer credential is presented or the server refuses it", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -3790,10 +3392,17 @@ }, "description": "Bearer challenge (RFC 6750 §3). It carries error=\"invalid_token\" when a presented bearer credential was refused, and resource_metadata naming the OAuth protected-resource metadata when the server has an OIDC resource identity." } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } } }, "403": { - "description": "Forbidden; HEAD responses have no body", + "description": "Forbidden", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -3804,10 +3413,17 @@ }, "description": "Supported OmniGraph HTTP API contract." } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } } }, "404": { - "description": "Unknown entity or null Blob cell; HEAD responses have no body; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "description": "Unknown entity or null Blob cell; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -3818,6 +3434,13 @@ }, "description": "Supported OmniGraph HTTP API contract." } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } } }, "405": { @@ -3838,10 +3461,17 @@ }, "description": "Supported OmniGraph HTTP API contract." } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } } }, "412": { - "description": "If-Match did not strongly match the selected managed Blob validator; HEAD responses have no body; actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", + "description": "If-Match did not strongly match the selected managed Blob validator; actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", "headers": { "Accept-Ranges": { "schema": { @@ -3867,11 +3497,34 @@ "type": "string" } } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } } }, - "429": { - "description": "too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", + "416": { + "description": "Requested managed byte range is unsatisfiable", "headers": { + "Accept-Ranges": { + "schema": { + "type": "string" + } + }, + "Content-Range": { + "schema": { + "type": "string" + }, + "description": "Unsatisfied range in the form `bytes */N`" + }, + "ETag": { + "schema": { + "type": "string" + } + }, "Omnigraph-Http-Api": { "schema": { "type": "string", @@ -3881,16 +3534,22 @@ }, "description": "Supported OmniGraph HTTP API contract." }, - "Retry-After": { + "Omnigraph-Snapshot-Id": { "schema": { "type": "string" - }, - "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." + } + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } } } }, - "500": { - "description": "Stored Blob integrity failure (code `internal`) or pre-header delivery refusal, including ranged external descriptors that cannot be redirected; HEAD responses have no body", + "429": { + "description": "too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -3900,10 +3559,44 @@ ] }, "description": "Supported OmniGraph HTTP API contract." - } - } - }, - "503": { + }, + "Retry-After": { + "schema": { + "type": "string" + }, + "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "500": { + "description": "Stored Blob integrity failure (code `internal`) or pre-header delivery refusal, including ranged external descriptors that cannot be redirected", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "503": { "description": "graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", "headers": { "Omnigraph-Http-Api": { @@ -3921,6 +3614,13 @@ }, "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } } } }, @@ -3929,16 +3629,14 @@ "bearer_token": [] } ] - } - }, - "/graphs/{graph_id}/branches": { - "get": { + }, + "put": { "tags": [ - "branches" + "blobs" ], - "summary": "List all branches.", - "description": "Returns branch names sorted by name in byte order. Read-only. The GQ statement\n`branch list` on `POST /query` runs the same body.", - "operationId": "branchList", + "summary": "Replace one Blob value of an existing node or edge with the request body.", + "description": "The cell's old value is never read; the entity's other cells are carried\nunchanged. Authorization of `change` on the branch runs before the body is\nread. Once admitted the write is owned by the server: a disconnect loses\nonly the response, never cancels or replays the write.", + "operationId": "blobPut", "parameters": [ { "name": "graph_id", @@ -3949,6 +3647,63 @@ "type": "string" } }, + { + "name": "entity", + "in": "query", + "description": "Select a logical node or edge cell.", + "required": true, + "schema": { + "$ref": "#/components/schemas/EntityKind" + } + }, + { + "name": "type", + "in": "query", + "description": "Accepted-schema node or edge type name.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "id", + "in": "query", + "description": "Logical id of an existing entity within the selected type.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "property", + "in": "query", + "description": "Accepted-schema Blob property name.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "branch", + "in": "query", + "description": "Branch to write. Defaults to `main`.", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "If-Match", + "in": "header", + "description": "`*` to require a value in the cell, or a list of strong entity tags one of which must equal the cell's current ETag. Weak tags never match. The write evaluates it against the branch head it applies to.", + "required": false, + "schema": { + "type": [ + "string", + "null" + ] + } + }, { "name": "Omnigraph-Http-Api", "in": "header", @@ -3971,10 +3726,29 @@ } } ], + "requestBody": { + "description": "The raw bytes to store: at most 33554432 bytes (32 MiB), inclusive.", + "content": { + "application/octet-stream": { + "schema": { + "type": "string", + "format": "binary", + "description": "OpenAPI-only marker for an unstructured octet-stream response body." + } + } + }, + "required": true + }, "responses": { "200": { - "description": "List of branches", + "description": "The bytes are published in one graph commit", "headers": { + "ETag": { + "schema": { + "type": "string" + }, + "description": "Strong validator of the stored value; equals `etag` in the body" + }, "Omnigraph-Http-Api": { "schema": { "type": "string", @@ -3988,13 +3762,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/BranchListOutput" + "$ref": "#/components/schemas/BlobWriteOutput" } } } }, "400": { - "description": "api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "description": "Invalid selector, query parameter, If-Match field or target, a non-Blob property, or a carried external reference the graph's policy denies; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -4063,7 +3837,7 @@ } }, "404": { - "description": "not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "description": "Unknown branch, or no entity with this id; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -4110,8 +3884,8 @@ } } }, - "412": { - "description": "actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", + "408": { + "description": "The request body did not arrive before the body deadline; request_timeout when the request body deadline passes before operation admission", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -4131,8 +3905,8 @@ } } }, - "429": { - "description": "too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", + "409": { + "description": "Write-authority conflict, or the branch's incarnation or accepted schema changed while the write retried", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -4142,12 +3916,6 @@ ] }, "description": "Supported OmniGraph HTTP API contract." - }, - "Retry-After": { - "schema": { - "type": "string" - }, - "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." } }, "content": { @@ -4158,9 +3926,15 @@ } } }, - "500": { - "description": "Internal failure (code `internal`) before a successful response", + "412": { + "description": "If-Match did not hold for the cell; `blob_precondition_failure` names its current validator; actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", "headers": { + "ETag": { + "schema": { + "type": "string" + }, + "description": "The cell's current validator, when it holds a managed value" + }, "Omnigraph-Http-Api": { "schema": { "type": "string", @@ -4179,8 +3953,8 @@ } } }, - "503": { - "description": "graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "413": { + "description": "The body exceeds the server's Blob put limit (the process `write_max_bytes`, at most 32 MiB), or the value with the entity's carried Blob payloads exceeds it; payload_too_large when the request body exceeds its route limit before operation admission", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -4190,12 +3964,6 @@ ] }, "description": "Supported OmniGraph HTTP API contract." - }, - "Retry-After": { - "schema": { - "type": "string" - }, - "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." } }, "content": { @@ -4205,66 +3973,9 @@ } } } - } - }, - "security": [ - { - "bearer_token": [] - } - ] - }, - "post": { - "tags": [ - "branches" - ], - "summary": "Create a new branch.", - "description": "Forks `name` off of `from` (defaults to `main`) at its head, or starts it\nat the published graph commit `from_commit` with that commit's data and\nhistory. The receipt names the branch forked and the commit the new\nbranch starts at. The new branch shares backing dataset data with its\nparent until it is mutated. Returns 409 if `name` already exists. The GQ\nstatement `branch create` on `POST /mutate` runs the same body.", - "operationId": "branchCreate", - "parameters": [ - { - "name": "graph_id", - "in": "path", - "description": "Graph id to route the request to. The `og_` prefix is reserved for system graphs. `og_catalog` is the built-in, storage-free catalogue: it serves only POST /query and GET /schema with their ordinary response bodies. Its query refuses `branch` and `snapshot`; every other route under /graphs/og_catalog returns 405 `method_not_allowed` before effects.", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "Omnigraph-Http-Api", - "in": "header", - "description": "exactly one Omnigraph-Http-Api header with value 0.14 is required", - "required": true, - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - } - }, - { - "name": "Omnigraph-If-Actor", - "in": "header", - "description": "The actor the caller expects its credential to resolve to. A mismatch is refused with 412 actor_mismatch before graph lookup or effects; a match changes nothing.", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/BranchCreateRequest" - } - } }, - "required": true - }, - "responses": { - "200": { - "description": "Branch created", + "415": { + "description": "Content-Type must be application/octet-stream", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -4279,13 +3990,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/BranchCreateOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } }, - "400": { - "description": "Bad request, or both `from` and `from_commit`; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "424": { + "description": "An allowed external Blob source carried from the entity could not be read", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -4305,8 +4016,8 @@ } } }, - "401": { - "description": "Unauthorized; unauthorized when no bearer credential is presented or the server refuses it", + "429": { + "description": "Per-actor admission cap exceeded; honor `Retry-After` header; too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -4317,11 +4028,11 @@ }, "description": "Supported OmniGraph HTTP API contract." }, - "WWW-Authenticate": { + "Retry-After": { "schema": { "type": "string" }, - "description": "Bearer challenge (RFC 6750 §3). It carries error=\"invalid_token\" when a presented bearer credential was refused, and resource_metadata naming the OAuth protected-resource metadata when the server has an OIDC resource identity." + "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." } }, "content": { @@ -4332,8 +4043,8 @@ } } }, - "403": { - "description": "Forbidden", + "500": { + "description": "Internal failure (code `internal`), or unsettled write completion (code `outcome_unknown`); follow the returned disposition before another write", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -4353,8 +4064,8 @@ } } }, - "404": { - "description": "`from` names no branch, or `from_commit` names no commit the actor may branch from; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "503": { + "description": "Write admission is closed or an overlapping durable recovery intent must be resolved before retry; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -4364,6 +4075,12 @@ ] }, "description": "Supported OmniGraph HTTP API contract." + }, + "Retry-After": { + "schema": { + "type": "string" + }, + "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." } }, "content": { @@ -4373,16 +4090,183 @@ } } } + } + }, + "security": [ + { + "bearer_token": [] + } + ] + }, + "delete": { + "tags": [ + "blobs" + ], + "summary": "Set one nullable Blob value of an existing node or edge to null.", + "description": "Clearing a cell that is already null publishes nothing. Like a put, the\nclear is owned by the server once admitted.", + "operationId": "blobClear", + "parameters": [ + { + "name": "graph_id", + "in": "path", + "description": "Graph id to route the request to. The `og_` prefix is reserved for system graphs. `og_catalog` is the built-in, storage-free catalogue: it serves only POST /query and GET /schema with their ordinary response bodies. Its query refuses `branch` and `snapshot`; every other route under /graphs/og_catalog returns 405 `method_not_allowed` before effects.", + "required": true, + "schema": { + "type": "string" + } }, - "405": { - "description": "method_not_allowed when the path is served, but not with this method", + { + "name": "entity", + "in": "query", + "description": "Select a logical node or edge cell.", + "required": true, + "schema": { + "$ref": "#/components/schemas/EntityKind" + } + }, + { + "name": "type", + "in": "query", + "description": "Accepted-schema node or edge type name.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "id", + "in": "query", + "description": "Logical id of an existing entity within the selected type.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "property", + "in": "query", + "description": "Accepted-schema Blob property name.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "branch", + "in": "query", + "description": "Branch to write. Defaults to `main`.", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "If-Match", + "in": "header", + "description": "`*` to require a value in the cell, or a list of strong entity tags one of which must equal the cell's current ETag. A null cell satisfies neither form.", + "required": false, + "schema": { + "type": [ + "string", + "null" + ] + } + }, + { + "name": "Omnigraph-Http-Api", + "in": "header", + "description": "exactly one Omnigraph-Http-Api header with value 0.14 is required", + "required": true, + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + } + }, + { + "name": "Omnigraph-If-Actor", + "in": "header", + "description": "The actor the caller expects its credential to resolve to. A mismatch is refused with 412 actor_mismatch before graph lookup or effects; a match changes nothing.", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "The cell is null. `commit` is the clear's publication, or `null` when the cell already was null and nothing was published", "headers": { - "Allow": { + "Omnigraph-Http-Api": { "schema": { - "type": "string" + "type": "string", + "enum": [ + "0.14" + ] }, - "description": "The methods the path serves." + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BlobWriteOutput" + } + } + } + }, + "400": { + "description": "Invalid selector, query parameter, If-Match field or target, a non-Blob or non-nullable property, or a carried external reference the graph's policy denies; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "401": { + "description": "Unauthorized; unauthorized when no bearer credential is presented or the server refuses it", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." }, + "WWW-Authenticate": { + "schema": { + "type": "string" + }, + "description": "Bearer challenge (RFC 6750 §3). It carries error=\"invalid_token\" when a presented bearer credential was refused, and resource_metadata naming the OAuth protected-resource metadata when the server has an OIDC resource identity." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "403": { + "description": "Forbidden", + "headers": { "Omnigraph-Http-Api": { "schema": { "type": "string", @@ -4401,8 +4285,8 @@ } } }, - "408": { - "description": "request_timeout when the request body deadline passes before operation admission", + "404": { + "description": "Unknown branch, or no entity with this id; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -4422,9 +4306,15 @@ } } }, - "409": { - "description": "Branch already exists, or the commit `from_commit` names cannot start a branch; see branch_start_refusal", + "405": { + "description": "method_not_allowed when the path is served, but not with this method", "headers": { + "Allow": { + "schema": { + "type": "string" + }, + "description": "The methods the path serves." + }, "Omnigraph-Http-Api": { "schema": { "type": "string", @@ -4443,8 +4333,8 @@ } } }, - "410": { - "description": "The commit `from_commit` names is no longer retained; see branch_start_refusal", + "409": { + "description": "Write-authority conflict, or the branch's incarnation or accepted schema changed while the write retried", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -4465,8 +4355,14 @@ } }, "412": { - "description": "actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", + "description": "If-Match did not hold for the cell; `blob_precondition_failure` names its current validator; actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", "headers": { + "ETag": { + "schema": { + "type": "string" + }, + "description": "The cell's current validator, when it holds a managed value" + }, "Omnigraph-Http-Api": { "schema": { "type": "string", @@ -4486,7 +4382,28 @@ } }, "413": { - "description": "payload_too_large when the request body exceeds its route limit before operation admission", + "description": "The entity's carried Blob payloads exceed the write's payload limit", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "424": { + "description": "An allowed external Blob source carried from the entity could not be read", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -4555,7 +4472,7 @@ } }, "503": { - "description": "An overlapping durable recovery intent must be resolved before retry; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "description": "Write admission is closed or an overlapping durable recovery intent must be resolved before retry; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -4587,16 +4504,14 @@ "bearer_token": [] } ] - } - }, - "/graphs/{graph_id}/branches/diff": { - "get": { + }, + "head": { "tags": [ - "changes" + "blobs" ], - "summary": "What a branch changed since its merge base with a target.", - "description": "The merge base is the commit `POST /branches/merge` would classify\nagainst; it and the source head are captured together. Changes the target\nmade after the base do not appear. Paged like `GET /diff`.", - "operationId": "branchDiff", + "summary": "Return the status and representation headers for one Blob cell.", + "description": "This is a distinct handler rather than Axum's automatic GET-to-HEAD\nfallback. It never calls `BlobReader::read_range`; Range and If-Range are\ndeliberately ignored while If-None-Match is still evaluated.", + "operationId": "blobStat", "parameters": [ { "name": "graph_id", @@ -4608,78 +4523,105 @@ } }, { - "name": "source", + "name": "entity", "in": "query", - "description": "Branch whose changes are reported.", + "description": "Select a logical node or edge cell.", + "required": true, + "schema": { + "$ref": "#/components/schemas/EntityKind" + } + }, + { + "name": "type", + "in": "query", + "description": "Accepted-schema node or edge type name.", "required": true, "schema": { "type": "string" } }, { - "name": "target", + "name": "id", "in": "query", - "description": "Branch the source is compared against. Defaults to `main`.", - "required": false, + "description": "Logical entity id within the selected type.", + "required": true, "schema": { "type": "string" } }, { - "name": "page_token", + "name": "property", "in": "query", - "description": "Opaque continuation from the preceding page of this response.", - "required": false, + "description": "Accepted-schema Blob property name.", + "required": true, "schema": { "type": "string" } }, { - "name": "limit", + "name": "branch", "in": "query", - "description": "Maximum changes per page, 1 to 8192; defaults to 1000. Above 8192\nthe request fails with 413; zero is refused with 400.", + "description": "Branch to read. Mutually exclusive with `snapshot`; defaults to `main`.", "required": false, "schema": { - "type": "integer", - "default": 1000, - "maximum": 8192, - "minimum": 1 + "type": "string" } }, { - "name": "kind", + "name": "snapshot", "in": "query", - "description": "Repeatable filter: node | edge.", + "description": "Immutable graph snapshot id. Mutually exclusive with `branch`.", "required": false, "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/EntityKind" - } + "type": "string" } }, { - "name": "type", - "in": "query", - "description": "Repeatable filter: accepted-schema type name.", + "name": "If-Match", + "in": "header", + "description": "Strong entity-tag-list precondition, including `*`, evaluated before If-None-Match.", "required": false, "schema": { - "type": "array", - "items": { - "type": "string" - } + "type": [ + "string", + "null" + ] } }, { - "name": "op", - "in": "query", - "description": "Repeatable filter: insert | update | delete.", + "name": "If-None-Match", + "in": "header", + "description": "Weak entity-tag-list comparison, including `*`. Range and If-Range are ignored for HEAD.", "required": false, "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/ChangeOp" - } + "type": [ + "string", + "null" + ] + } + }, + { + "name": "Range", + "in": "header", + "description": "Accepted but ignored for HEAD; metadata always describes the complete selected Blob.", + "required": false, + "schema": { + "type": [ + "string", + "null" + ] + } + }, + { + "name": "If-Range", + "in": "header", + "description": "Accepted but ignored for HEAD together with Range.", + "required": false, + "schema": { + "type": [ + "string", + "null" + ] } }, { @@ -4706,8 +4648,28 @@ ], "responses": { "200": { - "description": "What `source` changed since its merge base with `target`: the net entity changes from the base to the source head", + "description": "Managed Blob metadata with no response body", "headers": { + "Accept-Ranges": { + "schema": { + "type": "string" + }, + "description": "The literal value `bytes`" + }, + "Content-Length": { + "schema": { + "type": "integer", + "format": "int64", + "minimum": 0 + }, + "description": "Complete managed Blob length" + }, + "ETag": { + "schema": { + "type": "string" + }, + "description": "Strong validator for the selected managed Blob" + }, "Omnigraph-Http-Api": { "schema": { "type": "string", @@ -4716,19 +4678,30 @@ ] }, "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { + }, + "Omnigraph-Snapshot-Id": { "schema": { - "$ref": "#/components/schemas/DiffOutput" - } + "type": "string" + }, + "description": "Exact resolved graph snapshot" } } }, - "400": { - "description": "A missing source, an invalid filter or limit, or a rejected page token; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "302": { + "description": "External Blob descriptor; the server does not dereference it", "headers": { + "Cache-Control": { + "schema": { + "type": "string" + }, + "description": "The literal value `no-store`" + }, + "Location": { + "schema": { + "type": "string" + }, + "description": "Exact stored absolute URI" + }, "Omnigraph-Http-Api": { "schema": { "type": "string", @@ -4737,18 +4710,68 @@ ] }, "description": "Supported OmniGraph HTTP API contract." + }, + "Omnigraph-Snapshot-Id": { + "schema": { + "type": "string" + }, + "description": "Exact resolved graph snapshot" } - }, - "content": { - "application/json": { + } + }, + "304": { + "description": "If-None-Match matched the managed Blob validator", + "headers": { + "Accept-Ranges": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "type": "string" + } + }, + "Content-Length": { + "schema": { + "type": "integer", + "format": "int64", + "minimum": 0 + }, + "description": "Complete managed Blob length, as required for a valid 304 Content-Length" + }, + "ETag": { + "schema": { + "type": "string" + } + }, + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + }, + "Omnigraph-Snapshot-Id": { + "schema": { + "type": "string" } } } }, + "400": { + "description": "Invalid selector, target, or non-Blob property; HEAD responses have no body; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + } + }, "401": { - "description": "Unauthorized; unauthorized when no bearer credential is presented or the server refuses it", + "description": "Unauthorized; HEAD responses have no body; unauthorized when no bearer credential is presented or the server refuses it", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -4765,17 +4788,10 @@ }, "description": "Bearer challenge (RFC 6750 §3). It carries error=\"invalid_token\" when a presented bearer credential was refused, and resource_metadata naming the OAuth protected-resource metadata when the server has an OIDC resource identity." } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" - } - } } }, "403": { - "description": "The actor cannot read one of the branches", + "description": "Forbidden; HEAD responses have no body", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -4786,17 +4802,10 @@ }, "description": "Supported OmniGraph HTTP API contract." } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" - } - } } }, "404": { - "description": "A branch is not found; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "description": "Unknown entity or null Blob cell; HEAD responses have no body; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -4807,13 +4816,6 @@ }, "description": "Supported OmniGraph HTTP API contract." } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" - } - } } }, "405": { @@ -4834,18 +4836,21 @@ }, "description": "Supported OmniGraph HTTP API contract." } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" - } - } } }, - "409": { - "description": "The base and the source head cross a schema boundary; see change_diff_refusal", + "412": { + "description": "If-Match did not strongly match the selected managed Blob validator; HEAD responses have no body; actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", "headers": { + "Accept-Ranges": { + "schema": { + "type": "string" + } + }, + "ETag": { + "schema": { + "type": "string" + } + }, "Omnigraph-Http-Api": { "schema": { "type": "string", @@ -4854,75 +4859,10 @@ ] }, "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { + }, + "Omnigraph-Snapshot-Id": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" - } - } - } - }, - "410": { - "description": "Retained history for the base is no longer readable; see change_feed_gap", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" - } - } - } - }, - "412": { - "description": "actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" - } - } - } - }, - "413": { - "description": "Requested limit exceeds the public change ceiling", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "type": "string" } } } @@ -4945,17 +4885,10 @@ }, "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" - } - } } }, "500": { - "description": "Internal failure (code `internal`) while reading changes", + "description": "Stored Blob integrity failure (code `internal`) or pre-header delivery refusal, including ranged external descriptors that cannot be redirected; HEAD responses have no body", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -4966,17 +4899,10 @@ }, "description": "Supported OmniGraph HTTP API contract." } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" - } - } } }, "503": { - "description": "Recovery required before changes can be read; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "description": "graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -4993,13 +4919,6 @@ }, "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" - } - } } } }, @@ -5010,14 +4929,14 @@ ] } }, - "/graphs/{graph_id}/branches/merge": { - "post": { + "/graphs/{graph_id}/branches": { + "get": { "tags": [ "branches" ], - "summary": "Merge one branch into another.", - "description": "Merges `source` into `target` (defaults to `main`). Outcome is one of\n`already_up_to_date`, `fast_forward`, or `merged`. Returns 409 with the\nlist of conflicts if the merge cannot be completed; the target is left\nunchanged in that case. **Destructive** to `target` on success. `commit`\ncarries this merge's own target publication, including for a fast-forward;\nan already-up-to-date merge returns `commit: null`.\n\nWith `dry_run: true` the merge stops after classification and validation:\nthe response reports the outcome it would have now (`commit: null`,\n`dry_run: true`), a conflict answers the same 409, and nothing is written.\nA dry run is a read of both branches and needs no `branch_merge` grant.\n\nWith `delete_branch: true` the source branch is deleted after a successful\nmerge, under its own `branch_delete` policy check. The merge is durable by\nthen, so a deletion refusal or failure never fails the request; it is\nreported via `branch_deleted: false` + `branch_delete_error_details`.\n\nThe GQ statement `branch merge` on `POST /mutate` runs the same body\n(without the deletion composition) and answers the same 409 on conflict.", - "operationId": "branchMerge", + "summary": "List all branches.", + "description": "Returns branch names sorted by name in byte order. Read-only. The GQ statement\n`branch list` on `POST /query` runs the same body.", + "operationId": "branchList", "parameters": [ { "name": "graph_id", @@ -5050,19 +4969,9 @@ } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/BranchMergeRequest" - } - } - }, - "required": true - }, "responses": { "200": { - "description": "Branches merged", + "description": "List of branches", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -5077,13 +4986,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/BranchMergeOutput" + "$ref": "#/components/schemas/BranchListOutput" } } } }, "400": { - "description": "Bad request; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "description": "api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -5199,48 +5108,6 @@ } } }, - "408": { - "description": "request_timeout when the request body deadline passes before operation admission", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorOutput" - } - } - } - }, - "409": { - "description": "Merge conflict", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorOutput" - } - } - } - }, "412": { "description": "actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", "headers": { @@ -5262,50 +5129,8 @@ } } }, - "413": { - "description": "Merge entity, byte, or recovery-chain ceiling exceeded before effects; payload_too_large when the request body exceeds its route limit before operation admission", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorOutput" - } - } - } - }, - "424": { - "description": "A merge could not probe or read an allowed external Blob source", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorOutput" - } - } - } - }, "429": { - "description": "Per-actor admission cap exceeded; honor `Retry-After` header; too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", + "description": "too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -5332,7 +5157,7 @@ } }, "500": { - "description": "Internal failure (code `internal`), or unsettled write completion (code `outcome_unknown`); follow the returned disposition before another write", + "description": "Internal failure (code `internal`) before a successful response", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -5353,7 +5178,7 @@ } }, "503": { - "description": "An overlapping durable recovery intent must be resolved before retry; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "description": "graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -5385,16 +5210,14 @@ "bearer_token": [] } ] - } - }, - "/graphs/{graph_id}/branches/merge-base": { - "get": { + }, + "post": { "tags": [ - "changes" + "branches" ], - "summary": "The merge base of two branches.", - "description": "Reads lineage only, by the rule `POST /branches/merge` uses.", - "operationId": "branchMergeBase", + "summary": "Create a new branch.", + "description": "Forks `name` off of `from` (defaults to `main`) at its head, or starts it\nat the published graph commit `from_commit` with that commit's data and\nhistory. The receipt names the branch forked and the commit the new\nbranch starts at. The new branch shares backing dataset data with its\nparent until it is mutated. Returns 409 if `name` already exists. The GQ\nstatement `branch create` on `POST /mutate` runs the same body.", + "operationId": "branchCreate", "parameters": [ { "name": "graph_id", @@ -5405,24 +5228,6 @@ "type": "string" } }, - { - "name": "source", - "in": "query", - "description": "Branch that would be merged.", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "target", - "in": "query", - "description": "Branch that would receive the merge. Defaults to `main`.", - "required": false, - "schema": { - "type": "string" - } - }, { "name": "Omnigraph-Http-Api", "in": "header", @@ -5445,9 +5250,19 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BranchCreateRequest" + } + } + }, + "required": true + }, "responses": { "200": { - "description": "The commit a merge of `source` into `target` would classify against, with the two heads", + "description": "Branch created", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -5462,13 +5277,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/MergeBaseOutput" + "$ref": "#/components/schemas/BranchCreateOutput" } } } }, "400": { - "description": "A missing or unknown parameter; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "description": "Bad request, or both `from` and `from_commit`; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -5483,7 +5298,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } @@ -5510,13 +5325,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } }, "403": { - "description": "The actor cannot read one of the branches", + "description": "Forbidden", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -5531,13 +5346,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } }, "404": { - "description": "A branch is not found; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "description": "`from` names no branch, or `from_commit` names no commit the actor may branch from; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -5552,7 +5367,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } @@ -5579,7 +5394,70 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "408": { + "description": "request_timeout when the request body deadline passes before operation admission", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "409": { + "description": "Branch already exists, or the commit `from_commit` names cannot start a branch; see branch_start_refusal", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "410": { + "description": "The commit `from_commit` names is no longer retained; see branch_start_refusal", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" } } } @@ -5600,13 +5478,34 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "413": { + "description": "payload_too_large when the request body exceeds its route limit before operation admission", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" } } } }, "429": { - "description": "too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", + "description": "Per-actor admission cap exceeded; honor `Retry-After` header; too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -5627,13 +5526,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } }, "500": { - "description": "Internal failure (code `internal`) while reading lineage", + "description": "Internal failure (code `internal`), or unsettled write completion (code `outcome_unknown`); follow the returned disposition before another write", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -5648,13 +5547,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } }, "503": { - "description": "Recovery required before lineage can be read; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "description": "An overlapping durable recovery intent must be resolved before retry; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -5675,7 +5574,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } @@ -5688,14 +5587,14 @@ ] } }, - "/graphs/{graph_id}/branches/{branch_name}": { - "delete": { + "/graphs/{graph_id}/branches/diff": { + "get": { "tags": [ - "branches" + "changes" ], - "summary": "Delete a branch.", - "description": "**Irreversible.** Removes the branch pointer; commits remain reachable\nonly if referenced by another branch. Returns 404 if the branch does not\nexist. The GQ statement `branch delete` on `POST /mutate` runs the same\nbody.", - "operationId": "branchDelete", + "summary": "What a branch changed since its merge base with a target.", + "description": "The merge base is the commit `POST /branches/merge` would classify\nagainst; it and the source head are captured together. Changes the target\nmade after the base do not appear. Paged like `GET /diff`.", + "operationId": "branchDiff", "parameters": [ { "name": "graph_id", @@ -5707,14 +5606,80 @@ } }, { - "name": "branch_name", - "in": "path", - "description": "Branch name to delete", + "name": "source", + "in": "query", + "description": "Branch whose changes are reported.", "required": true, "schema": { "type": "string" } }, + { + "name": "target", + "in": "query", + "description": "Branch the source is compared against. Defaults to `main`.", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "page_token", + "in": "query", + "description": "Opaque continuation from the preceding page of this response.", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "limit", + "in": "query", + "description": "Maximum changes per page, 1 to 8192; defaults to 1000. Above 8192\nthe request fails with 413; zero is refused with 400.", + "required": false, + "schema": { + "type": "integer", + "default": 1000, + "maximum": 8192, + "minimum": 1 + } + }, + { + "name": "kind", + "in": "query", + "description": "Repeatable filter: node | edge.", + "required": false, + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/EntityKind" + } + } + }, + { + "name": "type", + "in": "query", + "description": "Repeatable filter: accepted-schema type name.", + "required": false, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "op", + "in": "query", + "description": "Repeatable filter: insert | update | delete.", + "required": false, + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ChangeOp" + } + } + }, { "name": "Omnigraph-Http-Api", "in": "header", @@ -5739,7 +5704,7 @@ ], "responses": { "200": { - "description": "Branch deleted", + "description": "What `source` changed since its merge base with `target`: the net entity changes from the base to the source head", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -5754,13 +5719,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/BranchDeleteOutput" + "$ref": "#/components/schemas/DiffOutput" } } } }, "400": { - "description": "api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "description": "A missing source, an invalid filter or limit, or a rejected page token; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -5775,7 +5740,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } @@ -5802,13 +5767,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, "403": { - "description": "Forbidden", + "description": "The actor cannot read one of the branches", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -5823,13 +5788,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, "404": { - "description": "Branch not found; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "description": "A branch is not found; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -5844,7 +5809,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } @@ -5871,13 +5836,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, - "412": { - "description": "actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", + "409": { + "description": "The base and the source head cross a schema boundary; see change_diff_refusal", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -5892,13 +5857,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, - "429": { - "description": "Per-actor admission cap exceeded; honor `Retry-After` header; too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", + "410": { + "description": "Retained history for the base is no longer readable; see change_feed_gap", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -5908,24 +5873,39 @@ ] }, "description": "Supported OmniGraph HTTP API contract." - }, - "Retry-After": { + } + }, + "content": { + "application/json": { "schema": { - "type": "string" + "$ref": "#/components/schemas/ChangeErrorOutput" + } + } + } + }, + "412": { + "description": "actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] }, - "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." + "description": "Supported OmniGraph HTTP API contract." } }, "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, - "500": { - "description": "Internal failure (code `internal`), or unsettled write completion (code `outcome_unknown`); follow the returned disposition before another write", + "413": { + "description": "Requested limit exceeds the public change ceiling", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -5940,13 +5920,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, - "503": { - "description": "An overlapping durable recovery intent must be resolved before retry; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "429": { + "description": "too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -5967,7 +5947,55 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" + } + } + } + }, + "500": { + "description": "Internal failure (code `internal`) while reading changes", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ChangeErrorOutput" + } + } + } + }, + "503": { + "description": "Recovery required before changes can be read; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + }, + "Retry-After": { + "schema": { + "type": "string" + }, + "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ChangeErrorOutput" } } } @@ -5980,14 +6008,14 @@ ] } }, - "/graphs/{graph_id}/changes": { - "get": { + "/graphs/{graph_id}/branches/merge": { + "post": { "tags": [ - "changes" + "branches" ], - "summary": "Poll the change feed of one branch.", - "description": "At-least-once: retrying a cursor may replay the complete next commit, so\nconsumers apply blocks idempotently by `graph_commit_id` and persist the\nterminal cursor together with its blocks. The server holds no consumer\nstate. `set=` values are validated and select nothing in this release.", - "operationId": "changesPoll", + "summary": "Merge one branch into another.", + "description": "Merges `source` into `target` (defaults to `main`). Outcome is one of\n`already_up_to_date`, `fast_forward`, or `merged`. Returns 409 with the\nlist of conflicts if the merge cannot be completed; the target is left\nunchanged in that case. **Destructive** to `target` on success. `commit`\ncarries this merge's own target publication, including for a fast-forward;\nan already-up-to-date merge returns `commit: null`.\n\nWith `dry_run: true` the merge stops after classification and validation:\nthe response reports the outcome it would have now (`commit: null`,\n`dry_run: true`), a conflict answers the same 409, and nothing is written.\nA dry run is a read of both branches and needs no `branch_merge` grant.\n\nWith `delete_branch: true` the source branch is deleted after a successful\nmerge, under its own `branch_delete` policy check. The merge is durable by\nthen, so a deletion refusal or failure never fails the request; it is\nreported via `branch_deleted: false` + `branch_delete_error_details`.\n\nThe GQ statement `branch merge` on `POST /mutate` runs the same body\n(without the deletion composition) and answers the same 409 on conflict.", + "operationId": "branchMerge", "parameters": [ { "name": "graph_id", @@ -5998,102 +6026,6 @@ "type": "string" } }, - { - "name": "branch", - "in": "query", - "description": "Branch whose first-parent history is polled. Defaults to `main`.", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "cursor", - "in": "query", - "description": "Durable cursor from a prior terminal page. Mutually exclusive with\n`start` and `page_token`.", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "start", - "in": "query", - "description": "Explicit start mode: `now` (default) | `beginning` |\n`after:`. Mutually exclusive with `cursor` and `page_token`.", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "page_token", - "in": "query", - "description": "Continuation of one bounded poll (keeps its captured cut). Mutually\nexclusive with `cursor` and `start`.", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "limit", - "in": "query", - "description": "Maximum changes per page, 1 to 8192; defaults to 1000. Above 8192\nthe request fails with 413; zero is refused with 400.", - "required": false, - "schema": { - "type": "integer", - "default": 1000, - "maximum": 8192, - "minimum": 1 - } - }, - { - "name": "kind", - "in": "query", - "description": "Repeatable filter: node | edge.", - "required": false, - "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/EntityKind" - } - } - }, - { - "name": "type", - "in": "query", - "description": "Repeatable filter: accepted-schema type name.", - "required": false, - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - }, - { - "name": "op", - "in": "query", - "description": "Repeatable filter: insert | update | delete.", - "required": false, - "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/ChangeOp" - } - } - }, - { - "name": "set", - "in": "query", - "description": "Repeatable session setting, `name=value` in GQ spelling\n(`set=merge_lineage=off`); only `request`-scope settings are accepted.\nValues are validated as session settings; `engine` does not change\nchange-feed execution.", - "required": false, - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - }, { "name": "Omnigraph-Http-Api", "in": "header", @@ -6116,9 +6048,19 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BranchMergeRequest" + } + } + }, + "required": true + }, "responses": { "200": { - "description": "Change blocks in first-parent order. The durable cursor appears only on a terminal page, advanced only over complete commits; a mid-block page carries only next_page_token", + "description": "Branches merged", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -6133,13 +6075,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeFeedOutput" + "$ref": "#/components/schemas/BranchMergeOutput" } } } }, "400": { - "description": "Invalid start/filter combination, or a rejected cursor or page token; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "description": "Bad request; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -6154,7 +6096,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } @@ -6181,7 +6123,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } @@ -6202,13 +6144,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } }, "404": { - "description": "Branch not found; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "description": "not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -6223,7 +6165,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } @@ -6250,13 +6192,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } }, - "409": { - "description": "The feed crossed an unprovable schema boundary; see change_diff_refusal", + "408": { + "description": "request_timeout when the request body deadline passes before operation admission", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -6271,13 +6213,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } }, - "410": { - "description": "Feed gap: required history was reclaimed; reset via the baseline handshake", + "409": { + "description": "Merge conflict", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -6292,7 +6234,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } @@ -6313,13 +6255,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } }, "413": { - "description": "Requested limit exceeds the public change ceiling", + "description": "Merge entity, byte, or recovery-chain ceiling exceeded before effects; payload_too_large when the request body exceeds its route limit before operation admission", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -6334,13 +6276,34 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "424": { + "description": "A merge could not probe or read an allowed external Blob source", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" } } } }, "429": { - "description": "too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", + "description": "Per-actor admission cap exceeded; honor `Retry-After` header; too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -6361,13 +6324,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } }, "500": { - "description": "Internal failure (code `internal`) while reading changes", + "description": "Internal failure (code `internal`), or unsettled write completion (code `outcome_unknown`); follow the returned disposition before another write", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -6382,13 +6345,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } }, "503": { - "description": "Recovery required before changes can be read; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "description": "An overlapping durable recovery intent must be resolved before retry; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -6409,7 +6372,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } @@ -6422,14 +6385,14 @@ ] } }, - "/graphs/{graph_id}/changes/baseline": { - "post": { + "/graphs/{graph_id}/branches/merge-base": { + "get": { "tags": [ "changes" ], - "summary": "Capture a change-feed baseline: one exact entity snapshot plus the cursor\nthat resumes the feed immediately after it.", - "description": "A baseline is a full data export, so it requires the export action. The\nsnapshot honors the scope's kind and type dimensions; `op` binds only the\nresume cursor's feed scope.", - "operationId": "changesBaseline", + "summary": "The merge base of two branches.", + "description": "Reads lineage only, by the rule `POST /branches/merge` uses.", + "operationId": "branchMergeBase", "parameters": [ { "name": "graph_id", @@ -6441,15 +6404,33 @@ } }, { - "name": "Omnigraph-Http-Api", - "in": "header", - "description": "exactly one Omnigraph-Http-Api header with value 0.14 is required", + "name": "source", + "in": "query", + "description": "Branch that would be merged.", "required": true, "schema": { - "type": "string", - "enum": [ - "0.14" - ] + "type": "string" + } + }, + { + "name": "target", + "in": "query", + "description": "Branch that would receive the merge. Defaults to `main`.", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "Omnigraph-Http-Api", + "in": "header", + "description": "exactly one Omnigraph-Http-Api header with value 0.14 is required", + "required": true, + "schema": { + "type": "string", + "enum": [ + "0.14" + ] } }, { @@ -6462,19 +6443,9 @@ } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ChangeBaselineRequest" - } - } - }, - "required": true - }, "responses": { "200": { - "description": "NDJSON entity snapshot pinned at one captured commit. Every preceding record is one type-keyed entity record (the load/export NDJSON shape); the FINAL record is the ChangeBaselineRecord envelope — an interrupted stream has no terminal record and therefore no usable cursor. Install the snapshot durably before the cursor.", + "description": "The commit a merge of `source` into `target` would classify against, with the two heads", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -6487,15 +6458,15 @@ } }, "content": { - "application/x-ndjson": { + "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeBaselineRecord" + "$ref": "#/components/schemas/MergeBaseOutput" } } } }, "400": { - "description": "Invalid scope; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "description": "A missing or unknown parameter; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -6543,7 +6514,7 @@ } }, "403": { - "description": "Forbidden", + "description": "The actor cannot read one of the branches", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -6564,7 +6535,7 @@ } }, "404": { - "description": "Branch not found; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "description": "A branch is not found; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -6611,27 +6582,6 @@ } } }, - "408": { - "description": "request_timeout when the request body deadline passes before operation admission", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" - } - } - } - }, "412": { "description": "actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", "headers": { @@ -6653,27 +6603,6 @@ } } }, - "413": { - "description": "Baseline cut or transport capacity exhausted; payload_too_large when the request body exceeds its route limit before operation admission", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" - } - } - } - }, "429": { "description": "too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", "headers": { @@ -6702,7 +6631,7 @@ } }, "500": { - "description": "Internal failure (code `internal`) while capturing the baseline", + "description": "Internal failure (code `internal`) while reading lineage", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -6723,7 +6652,7 @@ } }, "503": { - "description": "Recovery required; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "description": "Recovery required before lineage can be read; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -6757,14 +6686,14 @@ ] } }, - "/graphs/{graph_id}/commits": { - "get": { + "/graphs/{graph_id}/branches/{branch_name}": { + "delete": { "tags": [ - "commits" + "branches" ], - "summary": "List commits, most recent first.", - "description": "`branch` selects which history to list: a named branch returns the history\nreachable from that branch's head (the main commits inherited up to the\nfork plus the branch-authored commits); omitting it returns `main`'s\nhistory. There is no cross-branch listing. Ordering is part of the\ncontract: newest first along the branch's first-parent chain, which is\nfalling generation and falling (graph-manifest version, created-at, commit\nid). Read-only.\n\n`limit` pages the history: each page names the head its listing resolved\n(`head_commit_id`) and continues with `next_page_token`, sent back as\n`page_token` with the same `branch`; later pages answer for that head's\nchain even after the branch moves. A page reads only the history it\nreturns. Without `limit` and `page_token` the whole history is one answer.\nA caller whose `read` does not cover every branch continues only from a\ncommit of `branch`'s history: one the branch still holds or a readable\nbranch published costs nothing more; any other is looked for down the\nbranch's history, which costs what the whole listing costs, and a commit\nthe history does not reach is refused (400).\n\n`listing=unpublished` lists instead the unpublished commits whose recorded\ntarget is `branch` (writes `as commit` awaiting `commit publish`), newest\nfirst; each carries `published: false`. `listing=unattached` lists instead\nthe live unattached snapshots whose recorded target is `branch` (writes\n`as unattached branch`), newest first, each with `kind: \"unattached\"`;\nnon-history listings refuse `limit` and `page_token` (400). The retired\n`unpublished` and `unattached` keys are rejected regardless of value.", - "operationId": "commitList", + "summary": "Delete a branch.", + "description": "**Irreversible.** Removes the branch pointer; commits remain reachable\nonly if referenced by another branch. Returns 404 if the branch does not\nexist. The GQ statement `branch delete` on `POST /mutate` runs the same\nbody.", + "operationId": "branchDelete", "parameters": [ { "name": "graph_id", @@ -6776,44 +6705,10 @@ } }, { - "name": "branch", - "in": "query", - "description": "Branch whose commits to list. Defaults to `main`.", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "listing", - "in": "query", - "description": "Collection to list: `history` (default), `unpublished` proposals, or\n`unattached` snapshots whose recorded target is `branch`, newest first.", - "required": false, - "schema": { - "oneOf": [ - { - "$ref": "#/components/schemas/CommitListing" - } - ], - "default": "history" - } - }, - { - "name": "limit", - "in": "query", - "description": "Commits per page of the branch's history, 1 to 1000; defaults to 100\nwhen `page_token` is supplied. The page names the head it resolved\nand a `next_page_token`. Without `limit` and\n`page_token` the whole history is listed. Refused with a `listing` other\nthan `history`; above 1000 the request fails with 413.", - "required": false, - "schema": { - "type": "integer", - "maximum": 1000, - "minimum": 1 - } - }, - { - "name": "page_token", - "in": "query", - "description": "Opaque continuation from the preceding page of this listing, sent with\nthe same `branch`; 100 commits per page unless `limit` is set. Later\npages answer for the head the first page resolved.", - "required": false, + "name": "branch_name", + "in": "path", + "description": "Branch name to delete", + "required": true, "schema": { "type": "string" } @@ -6842,7 +6737,7 @@ ], "responses": { "200": { - "description": "List of commits", + "description": "Branch deleted", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -6857,13 +6752,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CommitListOutput" + "$ref": "#/components/schemas/BranchDeleteOutput" } } } }, "400": { - "description": "Invalid `listing`, a retired `unpublished` or `unattached` selector, paging a non-history listing, a zero `limit`, or a rejected page token; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "description": "api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -7000,29 +6895,8 @@ } } }, - "413": { - "description": "Requested limit exceeds 1000 commits per page", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorOutput" - } - } - } - }, "429": { - "description": "too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", + "description": "Per-actor admission cap exceeded; honor `Retry-After` header; too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -7049,7 +6923,7 @@ } }, "500": { - "description": "Internal failure (code `internal`) before a successful response", + "description": "Internal failure (code `internal`), or unsettled write completion (code `outcome_unknown`); follow the returned disposition before another write", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -7070,7 +6944,7 @@ } }, "503": { - "description": "graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "description": "An overlapping durable recovery intent must be resolved before retry; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -7104,14 +6978,14 @@ ] } }, - "/graphs/{graph_id}/commits/{commit_id}": { + "/graphs/{graph_id}/changes": { "get": { "tags": [ - "commits" + "changes" ], - "summary": "Get a single commit.", - "description": "Returns the commit's graph-manifest version, parent commit(s), and creation\nmetadata. Read-only.", - "operationId": "commitShow", + "summary": "Poll the change feed of one branch.", + "description": "At-least-once: retrying a cursor may replay the complete next commit, so\nconsumers apply blocks idempotently by `graph_commit_id` and persist the\nterminal cursor together with its blocks. The server holds no consumer\nstate. `set=` values are validated and select nothing in this release.", + "operationId": "changesPoll", "parameters": [ { "name": "graph_id", @@ -7123,75 +6997,162 @@ } }, { - "name": "commit_id", - "in": "path", - "description": "Commit identifier", - "required": true, + "name": "branch", + "in": "query", + "description": "Branch whose first-parent history is polled. Defaults to `main`.", + "required": false, "schema": { "type": "string" } }, { - "name": "Omnigraph-Http-Api", - "in": "header", - "description": "exactly one Omnigraph-Http-Api header with value 0.14 is required", - "required": true, + "name": "cursor", + "in": "query", + "description": "Durable cursor from a prior terminal page. Mutually exclusive with\n`start` and `page_token`.", + "required": false, "schema": { - "type": "string", - "enum": [ - "0.14" - ] + "type": "string" } }, { - "name": "Omnigraph-If-Actor", - "in": "header", - "description": "The actor the caller expects its credential to resolve to. A mismatch is refused with 412 actor_mismatch before graph lookup or effects; a match changes nothing.", + "name": "start", + "in": "query", + "description": "Explicit start mode: `now` (default) | `beginning` |\n`after:`. Mutually exclusive with `cursor` and `page_token`.", "required": false, "schema": { "type": "string" } - } - ], - "responses": { - "200": { - "description": "Commit details", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CommitOutput" - } - } + }, + { + "name": "page_token", + "in": "query", + "description": "Continuation of one bounded poll (keeps its captured cut). Mutually\nexclusive with `cursor` and `start`.", + "required": false, + "schema": { + "type": "string" } }, - "400": { - "description": "api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } + { + "name": "limit", + "in": "query", + "description": "Maximum changes per page, 1 to 8192; defaults to 1000. Above 8192\nthe request fails with 413; zero is refused with 400.", + "required": false, + "schema": { + "type": "integer", + "default": 1000, + "maximum": 8192, + "minimum": 1 + } + }, + { + "name": "kind", + "in": "query", + "description": "Repeatable filter: node | edge.", + "required": false, + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/EntityKind" + } + } + }, + { + "name": "type", + "in": "query", + "description": "Repeatable filter: accepted-schema type name.", + "required": false, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "op", + "in": "query", + "description": "Repeatable filter: insert | update | delete.", + "required": false, + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ChangeOp" + } + } + }, + { + "name": "set", + "in": "query", + "description": "Repeatable session setting, `name=value` in GQ spelling\n(`set=merge_lineage=off`); only `request`-scope settings are accepted.\nValues are validated as session settings; `engine` does not change\nchange-feed execution.", + "required": false, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "Omnigraph-Http-Api", + "in": "header", + "description": "exactly one Omnigraph-Http-Api header with value 0.14 is required", + "required": true, + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + } + }, + { + "name": "Omnigraph-If-Actor", + "in": "header", + "description": "The actor the caller expects its credential to resolve to. A mismatch is refused with 412 actor_mismatch before graph lookup or effects; a match changes nothing.", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Change blocks in first-parent order. The durable cursor appears only on a terminal page, advanced only over complete commits; a mid-block page carries only next_page_token", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } }, "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeFeedOutput" + } + } + } + }, + "400": { + "description": "Invalid start/filter combination, or a rejected cursor or page token; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ChangeErrorOutput" } } } @@ -7218,7 +7179,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } @@ -7239,13 +7200,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, "404": { - "description": "Commit not found; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "description": "Branch not found; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -7260,7 +7221,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } @@ -7287,7 +7248,49 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" + } + } + } + }, + "409": { + "description": "The feed crossed an unprovable schema boundary; see change_diff_refusal", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ChangeErrorOutput" + } + } + } + }, + "410": { + "description": "Feed gap: required history was reclaimed; reset via the baseline handshake", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ChangeErrorOutput" } } } @@ -7308,7 +7311,28 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" + } + } + } + }, + "413": { + "description": "Requested limit exceeds the public change ceiling", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ChangeErrorOutput" } } } @@ -7335,13 +7359,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, "500": { - "description": "Internal failure (code `internal`) before a successful response", + "description": "Internal failure (code `internal`) while reading changes", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -7356,13 +7380,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, "503": { - "description": "graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "description": "Recovery required before changes can be read; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -7383,7 +7407,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } @@ -7394,14 +7418,16 @@ "bearer_token": [] } ] - }, - "delete": { + } + }, + "/graphs/{graph_id}/changes/baseline": { + "post": { "tags": [ - "commits" + "changes" ], - "summary": "Drop an unpublished commit or an unattached snapshot.", - "description": "**Irreversible.** Runs under `commit_drop` on the commit's recorded target\nbranch; a denial answers the 404 an unknown id answers. A published commit\nis never dropped (409). Dropping an unattached snapshot also drops every\nunattached descendant, the root's whole chain included. The GQ statement\n`commit drop \"\"` on `POST /mutate` runs the same body.", - "operationId": "commitDrop", + "summary": "Capture a change-feed baseline: one exact entity snapshot plus the cursor\nthat resumes the feed immediately after it.", + "description": "A baseline is a full data export, so it requires the export action. The\nsnapshot honors the scope's kind and type dimensions; `op` binds only the\nresume cursor's feed scope.", + "operationId": "changesBaseline", "parameters": [ { "name": "graph_id", @@ -7412,15 +7438,6 @@ "type": "string" } }, - { - "name": "commit_id", - "in": "path", - "description": "Unpublished commit or unattached snapshot identifier", - "required": true, - "schema": { - "type": "string" - } - }, { "name": "Omnigraph-Http-Api", "in": "header", @@ -7443,9 +7460,19 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ChangeBaselineRequest" + } + } + }, + "required": true + }, "responses": { - "204": { - "description": "Commit dropped: the requested id and every unattached descendant lost addressability, and a proposal's held merge inputs are released", + "200": { + "description": "NDJSON entity snapshot pinned at one captured commit. Every preceding record is one type-keyed entity record (the load/export NDJSON shape); the FINAL record is the ChangeBaselineRecord envelope — an interrupted stream has no terminal record and therefore no usable cursor. Install the snapshot durably before the cursor.", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -7456,10 +7483,17 @@ }, "description": "Supported OmniGraph HTTP API contract." } + }, + "content": { + "application/x-ndjson": { + "schema": { + "$ref": "#/components/schemas/ChangeBaselineRecord" + } + } } }, "400": { - "description": "api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "description": "Invalid scope; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -7474,7 +7508,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } @@ -7501,14 +7535,14 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, - "404": { - "description": "Commit not found, or its drop is not permitted; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", - "headers": { + "403": { + "description": "Forbidden", + "headers": { "Omnigraph-Http-Api": { "schema": { "type": "string", @@ -7522,7 +7556,28 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" + } + } + } + }, + "404": { + "description": "Branch not found; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ChangeErrorOutput" } } } @@ -7549,13 +7604,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, - "409": { - "description": "The commit is published", + "408": { + "description": "request_timeout when the request body deadline passes before operation admission", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -7570,7 +7625,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } @@ -7591,13 +7646,34 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" + } + } + } + }, + "413": { + "description": "Baseline cut or transport capacity exhausted; payload_too_large when the request body exceeds its route limit before operation admission", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, "429": { - "description": "Per-actor admission cap exceeded; honor `Retry-After` header; too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", + "description": "too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -7618,13 +7694,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, "500": { - "description": "Internal failure (code `internal`), or unsettled write completion (code `outcome_unknown`); follow the returned disposition before another write", + "description": "Internal failure (code `internal`) while capturing the baseline", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -7639,13 +7715,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, "503": { - "description": "An overlapping durable recovery intent must be resolved before retry; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "description": "Recovery required; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -7666,7 +7742,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } @@ -7679,14 +7755,14 @@ ] } }, - "/graphs/{graph_id}/commits/{commit_id}/changes": { + "/graphs/{graph_id}/commits": { "get": { "tags": [ - "changes" + "commits" ], - "summary": "Entity changes one commit made relative to its first parent.", - "description": "Read-only, in graph vocabulary with exact before/after images. Bounded:\na large commit continues via the opaque `page_token`. `set=` values are\nvalidated and select nothing in this release.", - "operationId": "commitChanges", + "summary": "List commits, most recent first.", + "description": "`branch` selects which history to list: a named branch returns the history\nreachable from that branch's head (the main commits inherited up to the\nfork plus the branch-authored commits); omitting it returns `main`'s\nhistory. There is no cross-branch listing. Ordering is part of the\ncontract: newest first along the branch's first-parent chain, which is\nfalling generation and falling (graph-manifest version, created-at, commit\nid). Read-only.\n\n`limit` pages the history: each page names the head its listing resolved\n(`head_commit_id`) and continues with `next_page_token`, sent back as\n`page_token` with the same `branch`; later pages answer for that head's\nchain even after the branch moves. A page reads only the history it\nreturns. Without `limit` and `page_token` the whole history is one answer.\nA caller whose `read` does not cover every branch continues only from a\ncommit of `branch`'s history: one the branch still holds or a readable\nbranch published costs nothing more; any other is looked for down the\nbranch's history, which costs what the whole listing costs, and a commit\nthe history does not reach is refused (400).\n\n`listing=unpublished` lists instead the unpublished commits whose recorded\ntarget is `branch` (writes `as commit` awaiting `commit publish`), newest\nfirst; each carries `published: false`. `listing=unattached` lists instead\nthe live unattached snapshots whose recorded target is `branch` (writes\n`as unattached branch`), newest first, each with `kind: \"unattached\"`;\nnon-history listings refuse `limit` and `page_token` (400). The retired\n`unpublished` and `unattached` keys are rejected regardless of value.", + "operationId": "commitList", "parameters": [ { "name": "graph_id", @@ -7698,81 +7774,46 @@ } }, { - "name": "commit_id", - "in": "path", - "description": "Commit identifier", - "required": true, + "name": "branch", + "in": "query", + "description": "Branch whose commits to list. Defaults to `main`.", + "required": false, "schema": { "type": "string" } }, { - "name": "page_token", + "name": "listing", "in": "query", - "description": "Opaque continuation from the preceding page of this response.", + "description": "Collection to list: `history` (default), `unpublished` proposals, or\n`unattached` snapshots whose recorded target is `branch`, newest first.", "required": false, "schema": { - "type": "string" + "oneOf": [ + { + "$ref": "#/components/schemas/CommitListing" + } + ], + "default": "history" } }, { "name": "limit", "in": "query", - "description": "Maximum changes per page, 1 to 8192; defaults to 1000. Above 8192\nthe request fails with 413; zero is refused with 400.", + "description": "Commits per page of the branch's history, 1 to 1000; defaults to 100\nwhen `page_token` is supplied. The page names the head it resolved\nand a `next_page_token`. Without `limit` and\n`page_token` the whole history is listed. Refused with a `listing` other\nthan `history`; above 1000 the request fails with 413.", "required": false, "schema": { "type": "integer", - "default": 1000, - "maximum": 8192, + "maximum": 1000, "minimum": 1 } }, { - "name": "kind", - "in": "query", - "description": "Repeatable filter: node | edge.", - "required": false, - "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/EntityKind" - } - } - }, - { - "name": "type", - "in": "query", - "description": "Repeatable filter: accepted-schema type name.", - "required": false, - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - }, - { - "name": "op", - "in": "query", - "description": "Repeatable filter: insert | update | delete.", - "required": false, - "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/ChangeOp" - } - } - }, - { - "name": "set", + "name": "page_token", "in": "query", - "description": "Repeatable session setting, `name=value` in GQ spelling\n(`set=merge_lineage=off`); only `request`-scope settings are accepted.\nValues are validated as session settings; `engine` does not change\nchange-feed execution.", + "description": "Opaque continuation from the preceding page of this listing, sent with\nthe same `branch`; 100 commits per page unless `limit` is set. Later\npages answer for the head the first page resolved.", "required": false, "schema": { - "type": "array", - "items": { - "type": "string" - } + "type": "string" } }, { @@ -7799,7 +7840,7 @@ ], "responses": { "200": { - "description": "Entity changes this commit made relative to its first parent, in frozen (kind, type, id, op) order with the cause stated once", + "description": "List of commits", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -7814,13 +7855,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CommitChangesOutput" + "$ref": "#/components/schemas/CommitListOutput" } } } }, "400": { - "description": "Invalid filter or limit, or a rejected page token; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "description": "Invalid `listing`, a retired `unpublished` or `unattached` selector, paging a non-history listing, a zero `limit`, or a rejected page token; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -7835,7 +7876,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } @@ -7862,13 +7903,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } }, - "404": { - "description": "Commit not found, or the actor cannot read the commit's branch; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "403": { + "description": "Forbidden", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -7883,20 +7924,14 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } }, - "405": { - "description": "method_not_allowed when the path is served, but not with this method", + "404": { + "description": "Branch not found; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", "headers": { - "Allow": { - "schema": { - "type": "string" - }, - "description": "The methods the path serves." - }, "Omnigraph-Http-Api": { "schema": { "type": "string", @@ -7910,35 +7945,20 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } }, - "409": { - "description": "Commit cannot be entity-diffed (parentless commit or schema boundary); see change_diff_refusal", + "405": { + "description": "method_not_allowed when the path is served, but not with this method", "headers": { - "Omnigraph-Http-Api": { + "Allow": { "schema": { - "type": "string", - "enum": [ - "0.14" - ] + "type": "string" }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" - } - } - } - }, - "410": { - "description": "Required retained history is no longer readable; see change_feed_gap and capture a new baseline", - "headers": { + "description": "The methods the path serves." + }, "Omnigraph-Http-Api": { "schema": { "type": "string", @@ -7952,7 +7972,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } @@ -7973,13 +7993,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } }, "413": { - "description": "Requested limit exceeds the public change ceiling", + "description": "Requested limit exceeds 1000 commits per page", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -7994,7 +8014,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } @@ -8021,13 +8041,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } }, "500": { - "description": "Internal failure (code `internal`) while reading changes", + "description": "Internal failure (code `internal`) before a successful response", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -8042,13 +8062,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } }, "503": { - "description": "Recovery required before changes can be read; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "description": "graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -8069,7 +8089,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } @@ -8082,14 +8102,14 @@ ] } }, - "/graphs/{graph_id}/commits/{commit_id}/publish": { - "post": { + "/graphs/{graph_id}/commits/{commit_id}": { + "get": { "tags": [ "commits" ], - "summary": "Publish an unpublished commit onto its recorded target branch.", - "description": "The target is the branch the commit recorded when it was written `as\ncommit`, so the request carries no body. Runs under `commit_publish` on\nthat branch; a denial answers the 404 an unknown id answers. An\nunattached snapshot (`as unattached branch`) never publishes: 409 with\ncode `unattached_cannot_publish`, after the same authorization. The GQ\nstatement `commit publish \"\"` on `POST /mutate` runs the same body.", - "operationId": "commitPublish", + "summary": "Get a single commit.", + "description": "Returns the commit's graph-manifest version, parent commit(s), and creation\nmetadata. Read-only.", + "operationId": "commitShow", "parameters": [ { "name": "graph_id", @@ -8103,7 +8123,7 @@ { "name": "commit_id", "in": "path", - "description": "Unpublished commit identifier", + "description": "Commit identifier", "required": true, "schema": { "type": "string" @@ -8133,7 +8153,7 @@ ], "responses": { "200": { - "description": "The commit as published on its recorded target branch; a published id is answered as it stands", + "description": "Commit details", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -8201,8 +8221,8 @@ } } }, - "404": { - "description": "Commit not found, or its publication is not permitted; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "403": { + "description": "Forbidden", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -8222,15 +8242,9 @@ } } }, - "405": { - "description": "method_not_allowed when the path is served, but not with this method", + "404": { + "description": "Commit not found; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", "headers": { - "Allow": { - "schema": { - "type": "string" - }, - "description": "The methods the path serves." - }, "Omnigraph-Http-Api": { "schema": { "type": "string", @@ -8249,12 +8263,18 @@ } } }, - "409": { - "description": "The target's state moved under the proposal's authority (code `conflict`), or the commit is unattached and cannot publish (code `unattached_cannot_publish`, no durable effect)", + "405": { + "description": "method_not_allowed when the path is served, but not with this method", "headers": { - "Omnigraph-Http-Api": { + "Allow": { "schema": { - "type": "string", + "type": "string" + }, + "description": "The methods the path serves." + }, + "Omnigraph-Http-Api": { + "schema": { + "type": "string", "enum": [ "0.14" ] @@ -8271,7 +8291,7 @@ } }, "412": { - "description": "The target head moved past the commit's captured parent; the commit can only be dropped; actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", + "description": "actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -8292,7 +8312,7 @@ } }, "429": { - "description": "Per-actor admission cap exceeded; honor `Retry-After` header; too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", + "description": "too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -8319,7 +8339,7 @@ } }, "500": { - "description": "Internal failure (code `internal`), or unsettled write completion (code `outcome_unknown`); follow the returned disposition before another write", + "description": "Internal failure (code `internal`) before a successful response", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -8340,7 +8360,7 @@ } }, "503": { - "description": "An overlapping durable recovery intent must be resolved before retry; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "description": "graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -8372,16 +8392,14 @@ "bearer_token": [] } ] - } - }, - "/graphs/{graph_id}/diff": { - "get": { + }, + "delete": { "tags": [ - "changes" + "commits" ], - "summary": "Net entity changes between two read targets.", - "description": "`from` and `to` each name a branch (its head now) or a graph commit\n(published, an unpublished proposal, or an unattached snapshot); a string\nnaming both is refused. The first page resolves both sides to exact\ncommits and names them; a large result continues via the opaque\n`page_token`, and every page answers for those same commits.", - "operationId": "diff", + "summary": "Drop an unpublished commit or an unattached snapshot.", + "description": "**Irreversible.** Runs under `commit_drop` on the commit's recorded target\nbranch; a denial answers the 404 an unknown id answers. A published commit\nis never dropped (409). Dropping an unattached snapshot also drops every\nunattached descendant, the root's whole chain included. The GQ statement\n`commit drop \"\"` on `POST /mutate` runs the same body.", + "operationId": "commitDrop", "parameters": [ { "name": "graph_id", @@ -8393,80 +8411,14 @@ } }, { - "name": "from", - "in": "query", - "description": "Read target the diff starts from: a branch name (its head now) or a\ngraph commit id. A string naming both is refused.", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "to", - "in": "query", - "description": "Read target the diff ends at, spelled as `from`.", + "name": "commit_id", + "in": "path", + "description": "Unpublished commit or unattached snapshot identifier", "required": true, "schema": { "type": "string" } }, - { - "name": "page_token", - "in": "query", - "description": "Opaque continuation from the preceding page of this response.", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "limit", - "in": "query", - "description": "Maximum changes per page, 1 to 8192; defaults to 1000. Above 8192\nthe request fails with 413; zero is refused with 400.", - "required": false, - "schema": { - "type": "integer", - "default": 1000, - "maximum": 8192, - "minimum": 1 - } - }, - { - "name": "kind", - "in": "query", - "description": "Repeatable filter: node | edge.", - "required": false, - "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/EntityKind" - } - } - }, - { - "name": "type", - "in": "query", - "description": "Repeatable filter: accepted-schema type name.", - "required": false, - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - }, - { - "name": "op", - "in": "query", - "description": "Repeatable filter: insert | update | delete.", - "required": false, - "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/ChangeOp" - } - } - }, { "name": "Omnigraph-Http-Api", "in": "header", @@ -8490,8 +8442,8 @@ } ], "responses": { - "200": { - "description": "Net entity changes between the two resolved commits, in frozen (kind, type, id, op) order, without a cause", + "204": { + "description": "Commit dropped: the requested id and every unattached descendant lost addressability, and a proposal's held merge inputs are released", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -8502,17 +8454,10 @@ }, "description": "Supported OmniGraph HTTP API contract." } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/DiffOutput" - } - } } }, "400": { - "description": "A missing side, an invalid filter or limit, a side naming both a branch and a commit, or a rejected page token; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "description": "api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -8527,7 +8472,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } @@ -8554,13 +8499,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } }, "404": { - "description": "A side names no branch and no commit, or the actor cannot read the branch it resolves to; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "description": "Commit not found, or its drop is not permitted; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -8575,7 +8520,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } @@ -8602,34 +8547,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } }, "409": { - "description": "The two sides cross a schema boundary; see change_diff_refusal", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" - } - } - } - }, - "410": { - "description": "Retained history for an endpoint is no longer readable; see change_feed_gap", + "description": "The commit is published", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -8644,7 +8568,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } @@ -8665,34 +8589,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" - } - } - } - }, - "413": { - "description": "Requested limit exceeds the public change ceiling", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } }, "429": { - "description": "too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", + "description": "Per-actor admission cap exceeded; honor `Retry-After` header; too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -8713,13 +8616,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } }, "500": { - "description": "Internal failure (code `internal`) while reading changes", + "description": "Internal failure (code `internal`), or unsettled write completion (code `outcome_unknown`); follow the returned disposition before another write", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -8734,13 +8637,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } }, "503": { - "description": "Recovery required before changes can be read; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "description": "An overlapping durable recovery intent must be resolved before retry; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -8761,7 +8664,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeErrorOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } @@ -8774,14 +8677,14 @@ ] } }, - "/graphs/{graph_id}/export": { - "post": { + "/graphs/{graph_id}/commits/{commit_id}/changes": { + "get": { "tags": [ - "queries" + "changes" ], - "summary": "Stream the contents of a branch as NDJSON.", - "description": "Emits one JSON object per line (`application/x-ndjson`). Filter with\n`type_names` (node/edge type names); an empty list streams the entire branch.\nSuitable for large exports — the response is streamed, not buffered.\nRead-only.", - "operationId": "export", + "summary": "Entity changes one commit made relative to its first parent.", + "description": "Read-only, in graph vocabulary with exact before/after images. Bounded:\na large commit continues via the opaque `page_token`. `set=` values are\nvalidated and select nothing in this release.", + "operationId": "commitChanges", "parameters": [ { "name": "graph_id", @@ -8793,61 +8696,112 @@ } }, { - "name": "Omnigraph-Http-Api", - "in": "header", - "description": "exactly one Omnigraph-Http-Api header with value 0.14 is required", + "name": "commit_id", + "in": "path", + "description": "Commit identifier", "required": true, "schema": { - "type": "string", - "enum": [ - "0.14" - ] + "type": "string" } }, { - "name": "Omnigraph-If-Actor", - "in": "header", - "description": "The actor the caller expects its credential to resolve to. A mismatch is refused with 412 actor_mismatch before graph lookup or effects; a match changes nothing.", + "name": "page_token", + "in": "query", + "description": "Opaque continuation from the preceding page of this response.", "required": false, "schema": { "type": "string" } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ExportRequest" - } - } }, - "required": true - }, - "responses": { - "200": { - "description": "Exported data as NDJSON", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/x-ndjson": {} + { + "name": "limit", + "in": "query", + "description": "Maximum changes per page, 1 to 8192; defaults to 1000. Above 8192\nthe request fails with 413; zero is refused with 400.", + "required": false, + "schema": { + "type": "integer", + "default": 1000, + "maximum": 8192, + "minimum": 1 } }, - "400": { - "description": "Bad request; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", + { + "name": "kind", + "in": "query", + "description": "Repeatable filter: node | edge.", + "required": false, + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/EntityKind" + } + } + }, + { + "name": "type", + "in": "query", + "description": "Repeatable filter: accepted-schema type name.", + "required": false, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "op", + "in": "query", + "description": "Repeatable filter: insert | update | delete.", + "required": false, + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ChangeOp" + } + } + }, + { + "name": "set", + "in": "query", + "description": "Repeatable session setting, `name=value` in GQ spelling\n(`set=merge_lineage=off`); only `request`-scope settings are accepted.\nValues are validated as session settings; `engine` does not change\nchange-feed execution.", + "required": false, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "Omnigraph-Http-Api", + "in": "header", + "description": "exactly one Omnigraph-Http-Api header with value 0.14 is required", + "required": true, + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + } + }, + { + "name": "Omnigraph-If-Actor", + "in": "header", + "description": "The actor the caller expects its credential to resolve to. A mismatch is refused with 412 actor_mismatch before graph lookup or effects; a match changes nothing.", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Entity changes this commit made relative to its first parent, in frozen (kind, type, id, op) order with the cause stated once", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", "enum": [ "0.14" ] @@ -8858,13 +8812,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/CommitChangesOutput" } } } }, - "401": { - "description": "Unauthorized; unauthorized when no bearer credential is presented or the server refuses it", + "400": { + "description": "Invalid filter or limit, or a rejected page token; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -8874,24 +8828,18 @@ ] }, "description": "Supported OmniGraph HTTP API contract." - }, - "WWW-Authenticate": { - "schema": { - "type": "string" - }, - "description": "Bearer challenge (RFC 6750 §3). It carries error=\"invalid_token\" when a presented bearer credential was refused, and resource_metadata naming the OAuth protected-resource metadata when the server has an OIDC resource identity." } }, "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, - "403": { - "description": "Forbidden", + "401": { + "description": "Unauthorized; unauthorized when no bearer credential is presented or the server refuses it", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -8901,18 +8849,24 @@ ] }, "description": "Supported OmniGraph HTTP API contract." + }, + "WWW-Authenticate": { + "schema": { + "type": "string" + }, + "description": "Bearer challenge (RFC 6750 §3). It carries error=\"invalid_token\" when a presented bearer credential was refused, and resource_metadata naming the OAuth protected-resource metadata when the server has an OIDC resource identity." } }, "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, "404": { - "description": "Branch not found; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "description": "Commit not found, or the actor cannot read the commit's branch; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -8927,7 +8881,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } @@ -8954,13 +8908,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, - "408": { - "description": "request_timeout when the request body deadline passes before operation admission", + "409": { + "description": "Commit cannot be entity-diffed (parentless commit or schema boundary); see change_diff_refusal", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -8975,13 +8929,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, - "409": { - "description": "Export authority conflict", + "410": { + "description": "Required retained history is no longer readable; see change_feed_gap and capture a new baseline", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -8996,7 +8950,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } @@ -9017,34 +8971,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, "413": { - "description": "Export cut or transport capacity exhausted; payload_too_large when the request body exceeds its route limit before operation admission", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorOutput" - } - } - } - }, - "415": { - "description": "Request body must use application/json", + "description": "Requested limit exceeds the public change ceiling", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -9059,7 +8992,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } @@ -9086,13 +9019,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, "500": { - "description": "Internal failure (code `internal`) before a successful response", + "description": "Internal failure (code `internal`) while reading changes", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -9107,13 +9040,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, "503": { - "description": "Recovery required; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "description": "Recovery required before changes can be read; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -9134,7 +9067,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } @@ -9147,14 +9080,14 @@ ] } }, - "/graphs/{graph_id}/load": { + "/graphs/{graph_id}/commits/{commit_id}/publish": { "post": { "tags": [ - "mutations" + "commits" ], - "summary": "Compatibility-load NDJSON data through a JSON envelope.", - "description": "`data` is NDJSON with one record per line. `mode` controls behavior on\nexisting entities: `merge` upserts by id (default), `append` strictly inserts\nabsent ids, and `overwrite` replaces type data. Branch creation is opt-in by\npresence of `from`: with `from` set, a missing `branch` is created from\nit; without `from`, `branch` must already exist — a missing branch is a\n404, never an implicit fork. **Destructive** when `mode` is `overwrite`\nor when the load produces conflicting writes.", - "operationId": "load", + "summary": "Publish an unpublished commit onto its recorded target branch.", + "description": "The target is the branch the commit recorded when it was written `as\ncommit`, so the request carries no body. Runs under `commit_publish` on\nthat branch; a denial answers the 404 an unknown id answers. An\nunattached snapshot (`as unattached branch`) never publishes: 409 with\ncode `unattached_cannot_publish`, after the same authorization. The GQ\nstatement `commit publish \"\"` on `POST /mutate` runs the same body.", + "operationId": "commitPublish", "parameters": [ { "name": "graph_id", @@ -9165,6 +9098,15 @@ "type": "string" } }, + { + "name": "commit_id", + "in": "path", + "description": "Unpublished commit identifier", + "required": true, + "schema": { + "type": "string" + } + }, { "name": "Omnigraph-Http-Api", "in": "header", @@ -9187,19 +9129,9 @@ } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/IngestRequest" - } - } - }, - "required": true - }, "responses": { "200": { - "description": "Load results", + "description": "The commit as published on its recorded target branch; a published id is answered as it stands", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -9214,13 +9146,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/IngestOutput" + "$ref": "#/components/schemas/CommitOutput" } } } }, "400": { - "description": "Bad request; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "description": "api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -9267,29 +9199,8 @@ } } }, - "403": { - "description": "Forbidden", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorOutput" - } - } - } - }, "404": { - "description": "not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "description": "Commit not found, or its publication is not permitted; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -9336,8 +9247,8 @@ } } }, - "408": { - "description": "request_timeout when the request body deadline passes before operation admission", + "409": { + "description": "The target's state moved under the proposal's authority (code `conflict`), or the commit is unattached and cannot publish (code `unattached_cannot_publish`, no durable effect)", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -9357,8 +9268,8 @@ } } }, - "409": { - "description": "Prepared load authority changed before effects", + "412": { + "description": "The target head moved past the commit's captured parent; the commit can only be dropped; actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -9378,8 +9289,8 @@ } } }, - "412": { - "description": "actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", + "429": { + "description": "Per-actor admission cap exceeded; honor `Retry-After` header; too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -9389,6 +9300,12 @@ ] }, "description": "Supported OmniGraph HTTP API contract." + }, + "Retry-After": { + "schema": { + "type": "string" + }, + "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." } }, "content": { @@ -9399,8 +9316,8 @@ } } }, - "413": { - "description": "Load input or external Blob admission exceeds a bounded per-operation entity or byte ceiling; payload_too_large when the request body exceeds its route limit before operation admission", + "500": { + "description": "Internal failure (code `internal`), or unsettled write completion (code `outcome_unknown`); follow the returned disposition before another write", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -9420,8 +9337,8 @@ } } }, - "424": { - "description": "An allowed external Blob source could not be probed or read", + "503": { + "description": "An overlapping durable recovery intent must be resolved before retry; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -9431,81 +9348,12 @@ ] }, "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorOutput" - } - } - } - }, - "429": { - "description": "Per-actor admission cap exceeded; honor `Retry-After` header; too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - }, - "Retry-After": { - "schema": { - "type": "string" - }, - "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorOutput" - } - } - } - }, - "500": { - "description": "Internal failure (code `internal`), or unsettled write completion (code `outcome_unknown`); follow the returned disposition before another write", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorOutput" - } - } - } - }, - "503": { - "description": "An overlapping durable recovery intent must be resolved before retry; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - }, - "Retry-After": { - "schema": { - "type": "string" - }, - "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." + }, + "Retry-After": { + "schema": { + "type": "string" + }, + "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." } }, "content": { @@ -9524,14 +9372,14 @@ ] } }, - "/graphs/{graph_id}/load/ndjson": { - "post": { + "/graphs/{graph_id}/diff": { + "get": { "tags": [ - "mutations" + "changes" ], - "summary": "Load one strict, bounded graph-level NDJSON batch.", - "description": "Bearer authentication runs in middleware. This handler completes both\nbranch authorization checks before polling the raw body. A successful\nresponse describes logical schema declarations only and is returned after\nthe ordinary graph commit is visible.", - "operationId": "loadNdjson", + "summary": "Net entity changes between two read targets.", + "description": "`from` and `to` each name a branch (its head now) or a graph commit\n(published, an unpublished proposal, or an unattached snapshot); a string\nnaming both is refused. The first page resolves both sides to exact\ncommits and names them; a large result continues via the opaque\n`page_token`, and every page answers for those same commits.", + "operationId": "diff", "parameters": [ { "name": "graph_id", @@ -9543,30 +9391,78 @@ } }, { - "name": "branch", + "name": "from", "in": "query", - "description": "Target branch. Defaults to `main`. Without `from`, it must exist.", - "required": false, + "description": "Read target the diff starts from: a branch name (its head now) or a\ngraph commit id. A string naming both is refused.", + "required": true, "schema": { "type": "string" } }, { - "name": "from", + "name": "to", "in": "query", - "description": "Parent branch used to create a missing target branch.", + "description": "Read target the diff ends at, spelled as `from`.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "page_token", + "in": "query", + "description": "Opaque continuation from the preceding page of this response.", "required": false, "schema": { "type": "string" } }, { - "name": "mode", + "name": "limit", "in": "query", - "description": "How existing entities are handled. Defaults to `merge`.", + "description": "Maximum changes per page, 1 to 8192; defaults to 1000. Above 8192\nthe request fails with 413; zero is refused with 400.", "required": false, "schema": { - "$ref": "#/components/schemas/LoadMode" + "type": "integer", + "default": 1000, + "maximum": 8192, + "minimum": 1 + } + }, + { + "name": "kind", + "in": "query", + "description": "Repeatable filter: node | edge.", + "required": false, + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/EntityKind" + } + } + }, + { + "name": "type", + "in": "query", + "description": "Repeatable filter: accepted-schema type name.", + "required": false, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "op", + "in": "query", + "description": "Repeatable filter: insert | update | delete.", + "required": false, + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ChangeOp" + } } }, { @@ -9591,20 +9487,9 @@ } } ], - "requestBody": { - "description": "Strict raw graph-level NDJSON. Each nonblank line is exactly one node envelope {\"type\":\"\",\"id\":\"\",\"data\":{...}} or edge envelope {\"edge\":\"\",\"id\":\"\",\"from\":\"\",\"to\":\"\",\"data\":{...}}. `data` defaults to {} and holds user properties; the optional top-level `id` follows ordinary ID semantics. Legacy-vintage graphs also accept `data.id` as identity when top-level `id` is absent and refuse both placements together. Duplicate, unknown, reserved physical, and noncanonical supplied entity-ID members are refused.", - "content": { - "application/x-ndjson": { - "schema": { - "type": "string" - } - } - }, - "required": true - }, "responses": { "200": { - "description": "One committed graph-batch result", + "description": "Net entity changes between the two resolved commits, in frozen (kind, type, id, op) order, without a cause", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -9619,13 +9504,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/GraphBatchLoadOutput" + "$ref": "#/components/schemas/DiffOutput" } } } }, "400": { - "description": "Malformed query or graph batch; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "description": "A missing side, an invalid filter or limit, a side naming both a branch and a commit, or a rejected page token; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -9640,7 +9525,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } @@ -9667,34 +9552,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" - } - } - } - }, - "403": { - "description": "Forbidden", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, "404": { - "description": "Target branch missing without `from`; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "description": "A side names no branch and no commit, or the actor cannot read the branch it resolves to; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -9709,7 +9573,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } @@ -9736,13 +9600,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, - "408": { - "description": "request_timeout when the request body deadline passes before operation admission", + "409": { + "description": "The two sides cross a schema boundary; see change_diff_refusal", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -9757,13 +9621,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, - "409": { - "description": "Prepared load authority changed before effects", + "410": { + "description": "Retained history for an endpoint is no longer readable; see change_feed_gap", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -9778,7 +9642,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } @@ -9799,13 +9663,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, "413": { - "description": "Request, load, or external Blob admission exceeds a bounded ceiling; payload_too_large when the request body exceeds its route limit before operation admission", + "description": "Requested limit exceeds the public change ceiling", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -9820,13 +9684,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, - "415": { - "description": "Content-Type must be application/x-ndjson", + "429": { + "description": "too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -9836,18 +9700,24 @@ ] }, "description": "Supported OmniGraph HTTP API contract." + }, + "Retry-After": { + "schema": { + "type": "string" + }, + "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." } }, "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, - "424": { - "description": "An allowed external Blob source could not be probed or read", + "500": { + "description": "Internal failure (code `internal`) while reading changes", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -9862,13 +9732,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } }, - "429": { - "description": "Per-actor admission cap exceeded; honor `Retry-After` header; too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", + "503": { + "description": "Recovery required before changes can be read; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -9889,55 +9759,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorOutput" - } - } - } - }, - "500": { - "description": "Internal failure (code `internal`), or unsettled write completion (code `outcome_unknown`); follow the returned disposition before another write", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorOutput" - } - } - } - }, - "503": { - "description": "An overlapping durable recovery intent must be resolved before retry; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - }, - "Retry-After": { - "schema": { - "type": "string" - }, - "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorOutput" + "$ref": "#/components/schemas/ChangeErrorOutput" } } } @@ -9950,14 +9772,14 @@ ] } }, - "/graphs/{graph_id}/mutate": { + "/graphs/{graph_id}/export": { "post": { "tags": [ - "mutations" + "queries" ], - "summary": "Apply a GQ mutation to a branch (canonical mutation endpoint).", - "description": "Writes to the named `branch` (defaults to `main`). Mutations are atomic\nper call and produce a new commit. Returns counts of nodes and edges\naffected. **Destructive**: on success the branch is updated; rejected\nmutations may still acquire locks briefly. Returns 409 when the prepared\nwrite authority changes before effects.\n\nConditional callers use `POST /mutate/if-graph-commit`, which requires and\nvalidates `Omnigraph-If-Graph-Commit`. This endpoint rejects that header.\n\nPairs with `POST /query` (read-only).\n\nThe GQ statements `branch create`, `branch delete`, and `branch merge`\n(grammar: `BranchStmt` in `omnigraph-compiler`) are also served\nhere, with no `branch`, `name`, or `params`: each runs the handler body\nof its `/branches` route (same Cedar action, admission check, and errors,\nincluding the 409 of a conflicting merge) and answers a `ChangeOutput`\nwhose `outcome` names the effect. `branch list` and the branch control\nreads (`branch diff`, `branch merge-base`, `branch merge … dry run`) are\nrejected with 400; send them to `POST /query`.", - "operationId": "mutate", + "summary": "Stream the contents of a branch as NDJSON.", + "description": "Emits one JSON object per line (`application/x-ndjson`). Filter with\n`type_names` (node/edge type names); an empty list streams the entire branch.\nSuitable for large exports — the response is streamed, not buffered.\nRead-only.", + "operationId": "export", "parameters": [ { "name": "graph_id", @@ -9994,7 +9816,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeRequest" + "$ref": "#/components/schemas/ExportRequest" } } }, @@ -10002,7 +9824,7 @@ }, "responses": { "200": { - "description": "Mutation results", + "description": "Exported data as NDJSON", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -10015,15 +9837,11 @@ } }, "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ChangeOutput" - } - } + "application/x-ndjson": {} } }, "400": { - "description": "Bad request - also returned when `branch list` or a branch control read (`branch diff`, `branch merge-base`, `branch merge … dry run`) arrives here instead of POST /query, when a request target accompanies a branch statement, when a name or parameters accompany a branch statement, and when a commit precondition accompanies a branch statement; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "description": "Bad request; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -10092,7 +9910,7 @@ } }, "404": { - "description": "`branch delete` of a branch that does not exist, as DELETE /branches/{branch_name} answers; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "description": "Branch not found; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -10161,7 +9979,7 @@ } }, "409": { - "description": "Write-authority conflict; also `branch create` of a branch that already exists, and a conflicting `branch merge`, whose body carries `merge_conflicts`", + "description": "Export authority conflict", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -10203,7 +10021,7 @@ } }, "413": { - "description": "Keyed write exceeds the per-commit entity or byte ceiling; payload_too_large when the request body exceeds its route limit before operation admission", + "description": "Export cut or transport capacity exhausted; payload_too_large when the request body exceeds its route limit before operation admission", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -10223,8 +10041,8 @@ } } }, - "424": { - "description": "An allowed external Blob source could not be probed or read", + "415": { + "description": "Request body must use application/json", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -10245,7 +10063,7 @@ } }, "429": { - "description": "Per-actor admission cap exceeded; honor `Retry-After` header; too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", + "description": "too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -10272,7 +10090,7 @@ } }, "500": { - "description": "Internal failure (code `internal`), or unsettled write completion (code `outcome_unknown`); follow the returned disposition before another write", + "description": "Internal failure (code `internal`) before a successful response", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -10293,7 +10111,7 @@ } }, "503": { - "description": "An overlapping durable recovery intent must be resolved before retry; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "description": "Recovery required; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -10327,14 +10145,14 @@ ] } }, - "/graphs/{graph_id}/mutate/if-graph-commit": { + "/graphs/{graph_id}/load": { "post": { "tags": [ "mutations" ], - "summary": "Apply a mutation only while the branch still has the required graph head.", - "description": "This explicit conditional route requires `Omnigraph-If-Graph-Commit` and\nvalidates its precondition before mutation effects. `/mutate` rejects the\nprecondition header; conditional callers must use this route.", - "operationId": "mutateIfGraphCommit", + "summary": "Compatibility-load NDJSON data through a JSON envelope.", + "description": "`data` is NDJSON with one record per line. `mode` controls behavior on\nexisting entities: `merge` upserts by id (default), `append` strictly inserts\nabsent ids, and `overwrite` replaces type data. Branch creation is opt-in by\npresence of `from`: with `from` set, a missing `branch` is created from\nit; without `from`, `branch` must already exist — a missing branch is a\n404, never an implicit fork. **Destructive** when `mode` is `overwrite`\nor when the load produces conflicting writes.", + "operationId": "load", "parameters": [ { "name": "graph_id", @@ -10345,15 +10163,6 @@ "type": "string" } }, - { - "name": "Omnigraph-If-Graph-Commit", - "in": "header", - "description": "Required raw graph-head commit id. The mutation runs only while the branch's effective head still equals it.", - "required": true, - "schema": { - "type": "string" - } - }, { "name": "Omnigraph-Http-Api", "in": "header", @@ -10380,7 +10189,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeRequest" + "$ref": "#/components/schemas/IngestRequest" } } }, @@ -10388,7 +10197,7 @@ }, "responses": { "200": { - "description": "Conditional mutation results", + "description": "Load results", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -10403,13 +10212,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ChangeOutput" + "$ref": "#/components/schemas/IngestOutput" } } } }, "400": { - "description": "Missing, duplicate, malformed, or invalid request; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "description": "Bad request; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -10547,7 +10356,7 @@ } }, "409": { - "description": "Write-authority conflict", + "description": "Prepared load authority changed before effects", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -10568,7 +10377,7 @@ } }, "412": { - "description": "Graph-commit precondition failed; the write had no effect; actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", + "description": "actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -10589,7 +10398,7 @@ } }, "413": { - "description": "Keyed write exceeds the per-commit entity or byte ceiling; payload_too_large when the request body exceeds its route limit before operation admission", + "description": "Load input or external Blob admission exceeds a bounded per-operation entity or byte ceiling; payload_too_large when the request body exceeds its route limit before operation admission", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -10713,14 +10522,14 @@ ] } }, - "/graphs/{graph_id}/queries": { - "get": { + "/graphs/{graph_id}/load/ndjson": { + "post": { "tags": [ - "queries" + "mutations" ], - "summary": "List the graph's exposed stored queries as a typed tool catalog.", - "description": "Returns every stored query in the `queries:` registry, each\nwith its MCP tool name, read/mutate flag, description/instruction, and\ntyped parameters — enough for a client to register them as tools without\nfetching `.gq` source. Cluster-served graphs have no per-query expose flag,\nso the catalog lists them all. Read-gated; the catalog is graph-wide (branch\nindependent — `read` is authorized against `main`). **Not** Cedar-filtered\nper query yet, so it can list a query whose `invoke_query` the caller\nlacks (a known gap until per-query authorization lands).", - "operationId": "queriesList", + "summary": "Load one strict, bounded graph-level NDJSON batch.", + "description": "Bearer authentication runs in middleware. This handler completes both\nbranch authorization checks before polling the raw body. A successful\nresponse describes logical schema declarations only and is returned after\nthe ordinary graph commit is visible.", + "operationId": "loadNdjson", "parameters": [ { "name": "graph_id", @@ -10731,6 +10540,33 @@ "type": "string" } }, + { + "name": "branch", + "in": "query", + "description": "Target branch. Defaults to `main`. Without `from`, it must exist.", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "from", + "in": "query", + "description": "Parent branch used to create a missing target branch.", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "mode", + "in": "query", + "description": "How existing entities are handled. Defaults to `merge`.", + "required": false, + "schema": { + "$ref": "#/components/schemas/LoadMode" + } + }, { "name": "Omnigraph-Http-Api", "in": "header", @@ -10753,9 +10589,20 @@ } } ], + "requestBody": { + "description": "Strict raw graph-level NDJSON. Each nonblank line is exactly one node envelope {\"type\":\"\",\"id\":\"\",\"data\":{...}} or edge envelope {\"edge\":\"\",\"id\":\"\",\"from\":\"\",\"to\":\"\",\"data\":{...}}. `data` defaults to {} and holds user properties; the optional top-level `id` follows ordinary ID semantics. Legacy-vintage graphs also accept `data.id` as identity when top-level `id` is absent and refuse both placements together. Duplicate, unknown, reserved physical, and noncanonical supplied entity-ID members are refused.", + "content": { + "application/x-ndjson": { + "schema": { + "type": "string" + } + } + }, + "required": true + }, "responses": { "200": { - "description": "Stored-query catalog (every stored query, with typed params)", + "description": "One committed graph-batch result", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -10770,13 +10617,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/QueriesCatalogOutput" + "$ref": "#/components/schemas/GraphBatchLoadOutput" } } } }, "400": { - "description": "api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "description": "Malformed query or graph batch; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -10845,7 +10692,7 @@ } }, "404": { - "description": "not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "description": "Target branch missing without `from`; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -10892,6 +10739,48 @@ } } }, + "408": { + "description": "request_timeout when the request body deadline passes before operation admission", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "409": { + "description": "Prepared load authority changed before effects", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, "412": { "description": "actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", "headers": { @@ -10913,8 +10802,8 @@ } } }, - "429": { - "description": "too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", + "413": { + "description": "Request, load, or external Blob admission exceeds a bounded ceiling; payload_too_large when the request body exceeds its route limit before operation admission", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -10924,12 +10813,27 @@ ] }, "description": "Supported OmniGraph HTTP API contract." - }, - "Retry-After": { + } + }, + "content": { + "application/json": { "schema": { - "type": "string" + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "415": { + "description": "Content-Type must be application/x-ndjson", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] }, - "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." + "description": "Supported OmniGraph HTTP API contract." } }, "content": { @@ -10940,8 +10844,8 @@ } } }, - "500": { - "description": "Internal failure (code `internal`) before a successful response", + "424": { + "description": "An allowed external Blob source could not be probed or read", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -10961,8 +10865,8 @@ } } }, - "503": { - "description": "graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "429": { + "description": "Per-actor admission cap exceeded; honor `Retry-After` header; too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -10987,37 +10891,76 @@ } } } - } - }, - "security": [ - { - "bearer_token": [] - } - ] - } - }, - "/graphs/{graph_id}/queries/{query_name}": { - "post": { - "tags": [ - "queries" - ], - "summary": "Invoke a curated, server-side stored query by name.", - "description": "The query source comes from the graph's `queries:` registry, not the\nrequest body — callers send only runtime inputs (`params`, `branch`,\n`snapshot`). Gated by the `invoke_query` Cedar action at the boundary;\na stored *mutation* additionally passes the engine's `change` gate\n(double-gated). An actor **without** `invoke_query` cannot tell a denied\nquery from a missing one — both return the same 404, so the catalog\ncan't be probed without the grant. Once `invoke_query` is held, the\ninner `read`/`change` gate may surface a 403 for an existing query the\nactor can't run (the intended double-gate signal).", - "operationId": "queryInvoke", - "parameters": [ - { - "name": "graph_id", - "in": "path", - "description": "Graph id to route the request to. The `og_` prefix is reserved for system graphs. `og_catalog` is the built-in, storage-free catalogue: it serves only POST /query and GET /schema with their ordinary response bodies. Its query refuses `branch` and `snapshot`; every other route under /graphs/og_catalog returns 405 `method_not_allowed` before effects.", - "required": true, - "schema": { - "type": "string" - } }, - { - "name": "query_name", + "500": { + "description": "Internal failure (code `internal`), or unsettled write completion (code `outcome_unknown`); follow the returned disposition before another write", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "503": { + "description": "An overlapping durable recovery intent must be resolved before retry; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + }, + "Retry-After": { + "schema": { + "type": "string" + }, + "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + } + }, + "security": [ + { + "bearer_token": [] + } + ] + } + }, + "/graphs/{graph_id}/mutate": { + "post": { + "tags": [ + "mutations" + ], + "summary": "Apply a GQ mutation to a branch (canonical mutation endpoint).", + "description": "Writes to the named `branch` (defaults to `main`). Mutations are atomic\nper call and produce a new commit. Returns counts of nodes and edges\naffected. **Destructive**: on success the branch is updated; rejected\nmutations may still acquire locks briefly. Returns 409 when the prepared\nwrite authority changes before effects.\n\nConditional callers use `POST /mutate/if-graph-commit`, which requires and\nvalidates `Omnigraph-If-Graph-Commit`. This endpoint rejects that header.\n\nPairs with `POST /query` (read-only).\n\nThe GQ statements `branch create`, `branch delete`, and `branch merge`\n(grammar: `BranchStmt` in `omnigraph-compiler`) are also served\nhere, with no `branch`, `name`, or `params`: each runs the handler body\nof its `/branches` route (same Cedar action, admission check, and errors,\nincluding the 409 of a conflicting merge) and answers a `ChangeOutput`\nwhose `outcome` names the effect. `branch list` and the branch control\nreads (`branch diff`, `branch merge-base`, `branch merge … dry run`) are\nrejected with 400; send them to `POST /query`.", + "operationId": "mutate", + "parameters": [ + { + "name": "graph_id", "in": "path", - "description": "Stored query name (the registry key)", + "description": "Graph id to route the request to. The `og_` prefix is reserved for system graphs. `og_catalog` is the built-in, storage-free catalogue: it serves only POST /query and GET /schema with their ordinary response bodies. Its query refuses `branch` and `snapshot`; every other route under /graphs/og_catalog returns 405 `method_not_allowed` before effects.", "required": true, "schema": { "type": "string" @@ -11049,21 +10992,15 @@ "content": { "application/json": { "schema": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/components/schemas/InvokeStoredQueryRequest" - } - ] + "$ref": "#/components/schemas/ChangeRequest" } } - } + }, + "required": true }, "responses": { "200": { - "description": "Read envelope (ReadOutput) or mutation envelope (ChangeOutput), serialized untagged", + "description": "Mutation results", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -11078,13 +11015,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/InvokeStoredQueryOutput" + "$ref": "#/components/schemas/ChangeOutput" } } } }, "400": { - "description": "Bad request (param type error; snapshot on a stored mutation); api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "description": "Bad request - also returned when `branch list` or a branch control read (`branch diff`, `branch merge-base`, `branch merge … dry run`) arrives here instead of POST /query, when a request target accompanies a branch statement, when a name or parameters accompany a branch statement, and when a commit precondition accompanies a branch statement; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -11132,7 +11069,7 @@ } }, "403": { - "description": "Forbidden (the inner `change` gate for a stored mutation)", + "description": "Forbidden", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -11153,7 +11090,7 @@ } }, "404": { - "description": "Unknown stored query, or `invoke_query` denied — indistinguishable to a caller without the grant; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "description": "`branch delete` of a branch that does not exist, as DELETE /branches/{branch_name} answers; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -11222,7 +11159,7 @@ } }, "409": { - "description": "Stored mutation write-authority conflict, or a full-text index requires explicit rebuilding (full_text_index_rebuild_required) or has no segment at the snapshot (full_text_index_required); neither is cleared by retrying", + "description": "Write-authority conflict; also `branch create` of a branch that already exists, and a conflicting `branch merge`, whose body carries `merge_conflicts`", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -11264,7 +11201,7 @@ } }, "413": { - "description": "Stored keyed mutation exceeds the per-commit entity or byte ceiling; payload_too_large when the request body exceeds its route limit before operation admission", + "description": "Keyed write exceeds the per-commit entity or byte ceiling; payload_too_large when the request body exceeds its route limit before operation admission", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -11285,7 +11222,7 @@ } }, "424": { - "description": "A stored mutation could not probe or read an allowed external Blob source", + "description": "An allowed external Blob source could not be probed or read", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -11354,7 +11291,7 @@ } }, "503": { - "description": "A stored mutation is blocked by a durable recovery intent; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "description": "An overlapping durable recovery intent must be resolved before retry; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -11388,14 +11325,14 @@ ] } }, - "/graphs/{graph_id}/queries/{query_name}/if-graph-commit": { + "/graphs/{graph_id}/mutate/if-graph-commit": { "post": { "tags": [ - "queries" + "mutations" ], - "summary": "Invoke one stored mutation with a required graph-head precondition.", - "description": "A distinct path makes support observable before any mutation runs; older\nservers return 404 instead of ignoring an unknown conditional header.", - "operationId": "queryInvokeIfGraphCommit", + "summary": "Apply a mutation only while the branch still has the required graph head.", + "description": "This explicit conditional route requires `Omnigraph-If-Graph-Commit` and\nvalidates its precondition before mutation effects. `/mutate` rejects the\nprecondition header; conditional callers must use this route.", + "operationId": "mutateIfGraphCommit", "parameters": [ { "name": "graph_id", @@ -11406,19 +11343,10 @@ "type": "string" } }, - { - "name": "query_name", - "in": "path", - "description": "Stored mutation name (the registry key)", - "required": true, - "schema": { - "type": "string" - } - }, { "name": "Omnigraph-If-Graph-Commit", "in": "header", - "description": "Required raw graph-head commit id. The stored mutation runs only while the branch's effective head still equals it.", + "description": "Required raw graph-head commit id. The mutation runs only while the branch's effective head still equals it.", "required": true, "schema": { "type": "string" @@ -11450,21 +11378,15 @@ "content": { "application/json": { "schema": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/components/schemas/InvokeStoredQueryRequest" - } - ] + "$ref": "#/components/schemas/ChangeRequest" } } - } + }, + "required": true }, "responses": { "200": { - "description": "Stored conditional mutation result", + "description": "Conditional mutation results", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -11485,7 +11407,7 @@ } }, "400": { - "description": "Missing, duplicate, malformed, read-only, or invalid invocation; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "description": "Missing, duplicate, malformed, or invalid request; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -11533,7 +11455,7 @@ } }, "403": { - "description": "Forbidden (the inner `change` gate)", + "description": "Forbidden", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -11554,7 +11476,7 @@ } }, "404": { - "description": "Unknown stored mutation, or `invoke_query` denied; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "description": "not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -11623,7 +11545,7 @@ } }, "409": { - "description": "Stored mutation write-authority conflict", + "description": "Write-authority conflict", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -11644,7 +11566,7 @@ } }, "412": { - "description": "Stored mutation graph-commit precondition failed; the write had no effect; actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", + "description": "Graph-commit precondition failed; the write had no effect; actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -11665,7 +11587,7 @@ } }, "413": { - "description": "Stored keyed mutation exceeds the per-commit entity or byte ceiling; payload_too_large when the request body exceeds its route limit before operation admission", + "description": "Keyed write exceeds the per-commit entity or byte ceiling; payload_too_large when the request body exceeds its route limit before operation admission", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -11686,7 +11608,7 @@ } }, "424": { - "description": "A stored mutation could not probe or read an allowed external Blob source", + "description": "An allowed external Blob source could not be probed or read", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -11755,7 +11677,7 @@ } }, "503": { - "description": "A stored mutation is blocked by a durable recovery intent; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "description": "An overlapping durable recovery intent must be resolved before retry; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -11789,14 +11711,14 @@ ] } }, - "/graphs/{graph_id}/query": { - "post": { + "/graphs/{graph_id}/queries": { + "get": { "tags": [ "queries" ], - "summary": "Execute an inline read query.", - "description": "Designed for ad-hoc exploration and AI-agent tool-use: short field\nnames (`query`, `name`) match the CLI `-e` flag and the GQ `query`\nkeyword. Mutations (`insert`/`update`/`delete`) are rejected with 400\n-- use `POST /mutate` for writes. Select a branch or snapshot and obtain\nthe pinned graph-commit token with the response. Cedar authorizes Read for\nread queries and Change for `explain load csv`.\n\nThe GQ statement `branch list` is also served here, with no `branch`,\n`snapshot`, `name`, or `params`: it answers one result per branch (field\n`name`, byte order) under the same `read` check as `GET /branches`. So are\nthe branch control reads, each under its route's checks and errors:\n`branch diff [into ]` and `branch diff to `\n(one result per entity change, fields `kind`, `type`, `id`, `op`,\n`before`, `after`; the compared commits and the continuation token in\n`page`; `after page \"\"` and `limit ` continue and bound it),\n`branch merge-base [into ]` and\n`branch merge [into ] dry run` (one result each).\n`branch create`, `branch delete`, and `branch merge` are rejected with\n400; send them to `POST /mutate`.\n\nThe GQ statement `explain query …` is served here too: the query is not\nrun, and the result describes its v2 logical, physical and available\nDataFusion trees, one result per node (fields `tree`, `depth`, `node`, `detail`), followed by\n`plan` entries for the passes and the document's other fields, under the\nrequest's target and `params`. `explain load csv …` answers the accepted\nload plan the same way (`logical_initial`, `logical`, `physical`, then\n`plan` entries), under the request's `branch` alone. It requires the load's\nChange decision before probing the external source's metadata; it reads\nno payload byte and publishes nothing.", - "operationId": "query", + "summary": "List the graph's exposed stored queries as a typed tool catalog.", + "description": "Returns every stored query in the `queries:` registry, each\nwith its MCP tool name, read/mutate flag, description/instruction, and\ntyped parameters — enough for a client to register them as tools without\nfetching `.gq` source. Cluster-served graphs have no per-query expose flag,\nso the catalog lists them all. Read-gated; the catalog is graph-wide (branch\nindependent — `read` is authorized against `main`). **Not** Cedar-filtered\nper query yet, so it can list a query whose `invoke_query` the caller\nlacks (a known gap until per-query authorization lands).", + "operationId": "queriesList", "parameters": [ { "name": "graph_id", @@ -11829,19 +11751,9 @@ } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/QueryRequest" - } - } - }, - "required": true - }, "responses": { "200": { - "description": "Query results", + "description": "Stored-query catalog (every stored query, with typed params)", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -11856,13 +11768,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ReadOutput" + "$ref": "#/components/schemas/QueriesCatalogOutput" } } } }, "400": { - "description": "Bad request - also returned when the query body contains mutations (use POST /mutate, for write queries), when a control write statement (`branch create`, `branch delete`, `branch merge`) arrives here instead of POST /mutate, when a request target accompanies a branch statement, and when a name or parameters accompany a branch statement; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "description": "api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -11978,48 +11890,6 @@ } } }, - "408": { - "description": "request_timeout when the request body deadline passes before operation admission", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorOutput" - } - } - } - }, - "409": { - "description": "A full-text index requires explicit rebuilding (full_text_index_rebuild_required) or has no segment at the snapshot (full_text_index_required); neither is cleared by retrying", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorOutput" - } - } - } - }, "412": { "description": "actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", "headers": { @@ -12041,8 +11911,8 @@ } } }, - "413": { - "description": "payload_too_large when the request body exceeds its route limit before operation admission", + "429": { + "description": "too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -12052,6 +11922,12 @@ ] }, "description": "Supported OmniGraph HTTP API contract." + }, + "Retry-After": { + "schema": { + "type": "string" + }, + "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." } }, "content": { @@ -12062,8 +11938,8 @@ } } }, - "424": { - "description": "A load explain could not probe an allowed external source", + "500": { + "description": "Internal failure (code `internal`) before a successful response", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -12083,56 +11959,8 @@ } } }, - "429": { - "description": "too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - }, - "Retry-After": { - "schema": { - "type": "string" - }, - "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorOutput" - } - } - } - }, - "500": { - "description": "Internal failure (code `internal`) before a successful response", - "headers": { - "Omnigraph-Http-Api": { - "schema": { - "type": "string", - "enum": [ - "0.14" - ] - }, - "description": "Supported OmniGraph HTTP API contract." - } - }, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorOutput" - } - } - } - }, - "503": { - "description": "graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "503": { + "description": "graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -12166,14 +11994,14 @@ ] } }, - "/graphs/{graph_id}/schema": { - "get": { + "/graphs/{graph_id}/queries/{query_name}": { + "post": { "tags": [ - "schema" + "queries" ], - "summary": "Read the current schema source.", - "description": "Returns the project's schema as a single string in `.pg` source form.\nUseful for clients that want to introspect available types and properties\nbefore constructing GQ queries. Read-only.", - "operationId": "schemaShow", + "summary": "Invoke a curated, server-side stored query by name.", + "description": "The query source comes from the graph's `queries:` registry, not the\nrequest body — callers send only runtime inputs (`params`, `branch`,\n`snapshot`). Gated by the `invoke_query` Cedar action at the boundary;\na stored *mutation* additionally passes the engine's `change` gate\n(double-gated). An actor **without** `invoke_query` cannot tell a denied\nquery from a missing one — both return the same 404, so the catalog\ncan't be probed without the grant. Once `invoke_query` is held, the\ninner `read`/`change` gate may surface a 403 for an existing query the\nactor can't run (the intended double-gate signal).", + "operationId": "queryInvoke", "parameters": [ { "name": "graph_id", @@ -12184,6 +12012,15 @@ "type": "string" } }, + { + "name": "query_name", + "in": "path", + "description": "Stored query name (the registry key)", + "required": true, + "schema": { + "type": "string" + } + }, { "name": "Omnigraph-Http-Api", "in": "header", @@ -12206,9 +12043,25 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/InvokeStoredQueryRequest" + } + ] + } + } + } + }, "responses": { "200": { - "description": "Current schema source", + "description": "Read envelope (ReadOutput) or mutation envelope (ChangeOutput), serialized untagged", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -12223,13 +12076,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SchemaOutput" + "$ref": "#/components/schemas/InvokeStoredQueryOutput" } } } }, "400": { - "description": "api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "description": "Bad request (param type error; snapshot on a stored mutation); api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -12277,7 +12130,7 @@ } }, "403": { - "description": "Forbidden", + "description": "Forbidden (the inner `change` gate for a stored mutation)", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -12298,7 +12151,7 @@ } }, "404": { - "description": "not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "description": "Unknown stored query, or `invoke_query` denied — indistinguishable to a caller without the grant; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -12345,6 +12198,48 @@ } } }, + "408": { + "description": "request_timeout when the request body deadline passes before operation admission", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "409": { + "description": "Stored mutation write-authority conflict, or a full-text index requires explicit rebuilding (full_text_index_rebuild_required) or has no segment at the snapshot (full_text_index_required); neither is cleared by retrying", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, "412": { "description": "actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", "headers": { @@ -12366,8 +12261,50 @@ } } }, + "413": { + "description": "Stored keyed mutation exceeds the per-commit entity or byte ceiling; payload_too_large when the request body exceeds its route limit before operation admission", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "424": { + "description": "A stored mutation could not probe or read an allowed external Blob source", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, "429": { - "description": "too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", + "description": "Per-actor admission cap exceeded; honor `Retry-After` header; too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -12394,7 +12331,7 @@ } }, "500": { - "description": "Internal failure (code `internal`) before a successful response", + "description": "Internal failure (code `internal`), or unsettled write completion (code `outcome_unknown`); follow the returned disposition before another write", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -12415,7 +12352,7 @@ } }, "503": { - "description": "graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "description": "A stored mutation is blocked by a durable recovery intent; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -12449,14 +12386,14 @@ ] } }, - "/graphs/{graph_id}/snapshot": { - "get": { + "/graphs/{graph_id}/queries/{query_name}/if-graph-commit": { + "post": { "tags": [ - "snapshots" + "queries" ], - "summary": "Read the current snapshot of a branch.", - "description": "Returns the graph-manifest version plus per-dataset metadata (path,\npublished dataset version, entity count) for every backing dataset on the\nbranch. Defaults to `main` when `branch` is omitted. Read-only.", - "operationId": "snapshot", + "summary": "Invoke one stored mutation with a required graph-head precondition.", + "description": "A distinct path makes support observable before any mutation runs; older\nservers return 404 instead of ignoring an unknown conditional header.", + "operationId": "queryInvokeIfGraphCommit", "parameters": [ { "name": "graph_id", @@ -12468,10 +12405,19 @@ } }, { - "name": "branch", - "in": "query", - "description": "Branch whose snapshot to read. Defaults to `main`.", - "required": false, + "name": "query_name", + "in": "path", + "description": "Stored mutation name (the registry key)", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "Omnigraph-If-Graph-Commit", + "in": "header", + "description": "Required raw graph-head commit id. The stored mutation runs only while the branch's effective head still equals it.", + "required": true, "schema": { "type": "string" } @@ -12498,9 +12444,25 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/InvokeStoredQueryRequest" + } + ] + } + } + } + }, "responses": { "200": { - "description": "Graph snapshot", + "description": "Stored conditional mutation result", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -12515,13 +12477,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SnapshotOutput" + "$ref": "#/components/schemas/ChangeOutput" } } } }, "400": { - "description": "api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "description": "Missing, duplicate, malformed, read-only, or invalid invocation; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -12569,7 +12531,7 @@ } }, "403": { - "description": "Forbidden", + "description": "Forbidden (the inner `change` gate)", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -12590,7 +12552,7 @@ } }, "404": { - "description": "not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "description": "Unknown stored mutation, or `invoke_query` denied; not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -12637,8 +12599,8 @@ } } }, - "412": { - "description": "actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", + "408": { + "description": "request_timeout when the request body deadline passes before operation admission", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -12658,8 +12620,8 @@ } } }, - "429": { - "description": "too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", + "409": { + "description": "Stored mutation write-authority conflict", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -12669,12 +12631,6 @@ ] }, "description": "Supported OmniGraph HTTP API contract." - }, - "Retry-After": { - "schema": { - "type": "string" - }, - "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." } }, "content": { @@ -12685,8 +12641,8 @@ } } }, - "500": { - "description": "Internal failure (code `internal`) before a successful response", + "412": { + "description": "Stored mutation graph-commit precondition failed; the write had no effect; actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -12706,8 +12662,8 @@ } } }, - "503": { - "description": "graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "413": { + "description": "Stored keyed mutation exceeds the per-commit entity or byte ceiling; payload_too_large when the request body exceeds its route limit before operation admission", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -12717,12 +12673,6 @@ ] }, "description": "Supported OmniGraph HTTP API contract." - }, - "Retry-After": { - "schema": { - "type": "string" - }, - "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." } }, "content": { @@ -12732,35 +12682,31 @@ } } } - } - }, - "security": [ - { - "bearer_token": [] - } - ] - } - }, - "/healthz": { - "get": { - "tags": [ - "health" - ], - "summary": "Liveness probe.", - "description": "Returns server status and version. Unauthenticated; safe to call from any\ncaller. Use this to confirm the server is reachable before invoking other\nendpoints.", - "operationId": "health", - "responses": { - "200": { - "description": "Server is healthy", + }, + "424": { + "description": "A stored mutation could not probe or read an allowed external Blob source", "headers": { - "Cache-Control": { + "Omnigraph-Http-Api": { "schema": { "type": "string", "enum": [ - "no-store" + "0.14" ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" } - }, + } + } + }, + "429": { + "description": "Per-actor admission cap exceeded; honor `Retry-After` header; too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", + "headers": { "Omnigraph-Http-Api": { "schema": { "type": "string", @@ -12769,37 +12715,46 @@ ] }, "description": "Supported OmniGraph HTTP API contract." + }, + "Retry-After": { + "schema": { + "type": "string" + }, + "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." } }, "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/HealthOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } - } - } - }, - "head": { - "tags": [ - "health" - ], - "summary": "Liveness probe.", - "description": "Returns server status and version. Unauthenticated; safe to call from any\ncaller. Use this to confirm the server is reachable before invoking other\nendpoints.", - "operationId": "health_head", - "responses": { - "200": { - "description": "Server is healthy", + }, + "500": { + "description": "Internal failure (code `internal`), or unsettled write completion (code `outcome_unknown`); follow the returned disposition before another write", "headers": { - "Cache-Control": { + "Omnigraph-Http-Api": { "schema": { "type": "string", "enum": [ - "no-store" + "0.14" ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" } - }, + } + } + }, + "503": { + "description": "A stored mutation is blocked by a durable recovery intent; graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "headers": { "Omnigraph-Http-Api": { "schema": { "type": "string", @@ -12808,23 +12763,83 @@ ] }, "description": "Supported OmniGraph HTTP API contract." + }, + "Retry-After": { + "schema": { + "type": "string" + }, + "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } } } } - } + }, + "security": [ + { + "bearer_token": [] + } + ] } }, - "/readyz": { - "get": { + "/graphs/{graph_id}/query": { + "post": { "tags": [ - "health" + "queries" ], - "summary": "Readiness witness (RFC 0049).", - "description": "Unauthenticated, and therefore minimal: it reports whether this replica\nis serving or draining, the applied `config_digest` it booted from, the\nledger revision and CAS it read, and registry/ready/loading/blocked counts. Graph\nids stay behind authenticated catalog endpoints. Partial availability is\nready but degraded after loading finishes; loading, no available graph or\nshutdown returns 503. A valid\nempty registry is ready. `/healthz` reports liveness independently.", - "operationId": "readiness", + "summary": "Execute an inline read query.", + "description": "Designed for ad-hoc exploration and AI-agent tool-use: short field\nnames (`query`, `name`) match the CLI `-e` flag and the GQ `query`\nkeyword. Mutations (`insert`/`update`/`delete`) are rejected with 400\n-- use `POST /mutate` for writes. Select a branch or snapshot and obtain\nthe pinned graph-commit token with the response. Cedar authorizes Read for\nread queries and Change for `explain load csv`.\n\nThe GQ statement `branch list` is also served here, with no `branch`,\n`snapshot`, `name`, or `params`: it answers one result per branch (field\n`name`, byte order) under the same `read` check as `GET /branches`. So are\nthe branch control reads, each under its route's checks and errors:\n`branch diff [into ]` and `branch diff to `\n(one result per entity change, fields `kind`, `type`, `id`, `op`,\n`before`, `after`; the compared commits and the continuation token in\n`page`; `after page \"\"` and `limit ` continue and bound it),\n`branch merge-base [into ]` and\n`branch merge [into ] dry run` (one result each).\n`branch create`, `branch delete`, and `branch merge` are rejected with\n400; send them to `POST /mutate`.\n\nThe GQ statement `explain query …` is served here too: the query is not\nrun, and the result describes its v2 logical, physical and available\nDataFusion trees, one result per node (fields `tree`, `depth`, `node`, `detail`), followed by\n`plan` entries for the passes and the document's other fields, under the\nrequest's target and `params`. `explain load csv …` answers the accepted\nload plan the same way (`logical_initial`, `logical`, `physical`, then\n`plan` entries), under the request's `branch` alone. It requires the load's\nChange decision before probing the external source's metadata; it reads\nno payload byte and publishes nothing.", + "operationId": "query", + "parameters": [ + { + "name": "graph_id", + "in": "path", + "description": "Graph id to route the request to. The `og_` prefix is reserved for system graphs. `og_catalog` is the built-in, storage-free catalogue: it serves only POST /query and GET /schema with their ordinary response bodies. Its query refuses `branch` and `snapshot`; every other route under /graphs/og_catalog returns 405 `method_not_allowed` before effects.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "Omnigraph-Http-Api", + "in": "header", + "description": "exactly one Omnigraph-Http-Api header with value 0.14 is required", + "required": true, + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + } + }, + { + "name": "Omnigraph-If-Actor", + "in": "header", + "description": "The actor the caller expects its credential to resolve to. A mismatch is refused with 412 actor_mismatch before graph lookup or effects; a match changes nothing.", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/QueryRequest" + } + } + }, + "required": true + }, "responses": { "200": { - "description": "Serving", + "description": "Query results", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -12839,13 +12854,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ReadinessOutput" + "$ref": "#/components/schemas/ReadOutput" } } } }, - "503": { - "description": "Graphs loading, no graph available or stopping", + "400": { + "description": "Bad request - also returned when the query body contains mutations (use POST /mutate, for write queries), when a control write statement (`branch create`, `branch delete`, `branch merge`) arrives here instead of POST /mutate, when a request target accompanies a branch statement, and when a name or parameters accompany a branch statement; api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", "headers": { "Omnigraph-Http-Api": { "schema": { @@ -12860,83 +12875,1066 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ReadinessOutput" + "$ref": "#/components/schemas/ErrorOutput" } } } - } - } - } - } - }, - "components": { - "schemas": { - "AccessAction": { - "type": "string", - "description": "A graph action a policy decides, as the access route reports it.", - "enum": [ - "read", - "export", - "change", - "schema_apply", - "branch_create", - "branch_delete", - "branch_merge", - "commit_publish", - "commit_drop", - "invoke_query" - ] - }, - "AccessOutput": { - "type": "object", - "description": "What the caller's own credential may do on one graph, by the graph's\ninstalled policy. It names actions, never the rules that allow them, and\nsays nothing about any other actor.", - "required": [ - "graph_id", - "graph_wide", - "actions" - ], - "properties": { - "actions": { - "type": "array", - "items": { - "$ref": "#/components/schemas/AccessAction" - }, - "description": "With `branch`, the actions allowed on it, including `branch_delete`,\n`commit_publish`, `commit_drop` and `schema_apply` with it as their\ntarget. With `target_branch` as well, also `branch_create` and\n`branch_merge` from `branch` into it. Empty without `branch`." - }, - "actor_id": { - "type": [ - "string", - "null" - ], - "description": "The actor the credential resolves to. Absent only on a server that runs\nwithout authentication." - }, - "branch": { - "type": [ - "string", - "null" - ], - "description": "The branch asked about, when one was." - }, - "graph_id": { - "type": "string" }, - "graph_wide": { - "type": "array", - "items": { - "$ref": "#/components/schemas/AccessAction" + "401": { + "description": "Unauthorized; unauthorized when no bearer credential is presented or the server refuses it", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + }, + "WWW-Authenticate": { + "schema": { + "type": "string" + }, + "description": "Bearer challenge (RFC 6750 §3). It carries error=\"invalid_token\" when a presented bearer credential was refused, and resource_metadata naming the OAuth protected-resource metadata when the server has an OIDC resource identity." + } }, - "description": "Actions allowed with no branch named: the graph-scoped ones, and the\nbranch-scoped ones a single rule allows on every branch (a rule with no\nbranch scope, `branch_scope: any` or `target_branch_scope: any`).\nRules that cover every branch only together, one for protected\nbranches and one for the rest, allow each named branch but leave the\naction out of this list; read `actions` for a branch." + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } }, - "target_branch": { - "type": [ - "string", - "null" - ], - "description": "The target branch asked about, when one was." - } - } - }, - "Applicability": { + "403": { + "description": "Forbidden", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "404": { + "description": "not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "405": { + "description": "method_not_allowed when the path is served, but not with this method", + "headers": { + "Allow": { + "schema": { + "type": "string" + }, + "description": "The methods the path serves." + }, + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "408": { + "description": "request_timeout when the request body deadline passes before operation admission", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "409": { + "description": "A full-text index requires explicit rebuilding (full_text_index_rebuild_required) or has no segment at the snapshot (full_text_index_required); neither is cleared by retrying", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "412": { + "description": "actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "413": { + "description": "payload_too_large when the request body exceeds its route limit before operation admission", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "424": { + "description": "A load explain could not probe an allowed external source", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "429": { + "description": "too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + }, + "Retry-After": { + "schema": { + "type": "string" + }, + "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "500": { + "description": "Internal failure (code `internal`) before a successful response", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "503": { + "description": "graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + }, + "Retry-After": { + "schema": { + "type": "string" + }, + "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + } + }, + "security": [ + { + "bearer_token": [] + } + ] + } + }, + "/graphs/{graph_id}/schema": { + "get": { + "tags": [ + "schema" + ], + "summary": "Read the current schema source.", + "description": "Returns the project's schema as a single string in `.pg` source form.\nUseful for clients that want to introspect available types and properties\nbefore constructing GQ queries. Read-only.", + "operationId": "schemaShow", + "parameters": [ + { + "name": "graph_id", + "in": "path", + "description": "Graph id to route the request to. The `og_` prefix is reserved for system graphs. `og_catalog` is the built-in, storage-free catalogue: it serves only POST /query and GET /schema with their ordinary response bodies. Its query refuses `branch` and `snapshot`; every other route under /graphs/og_catalog returns 405 `method_not_allowed` before effects.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "Omnigraph-Http-Api", + "in": "header", + "description": "exactly one Omnigraph-Http-Api header with value 0.14 is required", + "required": true, + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + } + }, + { + "name": "Omnigraph-If-Actor", + "in": "header", + "description": "The actor the caller expects its credential to resolve to. A mismatch is refused with 412 actor_mismatch before graph lookup or effects; a match changes nothing.", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Current schema source", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SchemaOutput" + } + } + } + }, + "400": { + "description": "api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "401": { + "description": "Unauthorized; unauthorized when no bearer credential is presented or the server refuses it", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + }, + "WWW-Authenticate": { + "schema": { + "type": "string" + }, + "description": "Bearer challenge (RFC 6750 §3). It carries error=\"invalid_token\" when a presented bearer credential was refused, and resource_metadata naming the OAuth protected-resource metadata when the server has an OIDC resource identity." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "403": { + "description": "Forbidden", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "404": { + "description": "not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "405": { + "description": "method_not_allowed when the path is served, but not with this method", + "headers": { + "Allow": { + "schema": { + "type": "string" + }, + "description": "The methods the path serves." + }, + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "412": { + "description": "actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "429": { + "description": "too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + }, + "Retry-After": { + "schema": { + "type": "string" + }, + "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "500": { + "description": "Internal failure (code `internal`) before a successful response", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "503": { + "description": "graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + }, + "Retry-After": { + "schema": { + "type": "string" + }, + "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + } + }, + "security": [ + { + "bearer_token": [] + } + ] + } + }, + "/graphs/{graph_id}/snapshot": { + "get": { + "tags": [ + "snapshots" + ], + "summary": "Read the current snapshot of a branch.", + "description": "Returns the graph-manifest version plus per-dataset metadata (path,\npublished dataset version, entity count) for every backing dataset on the\nbranch. Defaults to `main` when `branch` is omitted. Read-only.", + "operationId": "snapshot", + "parameters": [ + { + "name": "graph_id", + "in": "path", + "description": "Graph id to route the request to. The `og_` prefix is reserved for system graphs. `og_catalog` is the built-in, storage-free catalogue: it serves only POST /query and GET /schema with their ordinary response bodies. Its query refuses `branch` and `snapshot`; every other route under /graphs/og_catalog returns 405 `method_not_allowed` before effects.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "branch", + "in": "query", + "description": "Branch whose snapshot to read. Defaults to `main`.", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "Omnigraph-Http-Api", + "in": "header", + "description": "exactly one Omnigraph-Http-Api header with value 0.14 is required", + "required": true, + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + } + }, + { + "name": "Omnigraph-If-Actor", + "in": "header", + "description": "The actor the caller expects its credential to resolve to. A mismatch is refused with 412 actor_mismatch before graph lookup or effects; a match changes nothing.", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Graph snapshot", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SnapshotOutput" + } + } + } + }, + "400": { + "description": "api_contract_mismatch when the required API contract header is missing, duplicated or unsupported", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "401": { + "description": "Unauthorized; unauthorized when no bearer credential is presented or the server refuses it", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + }, + "WWW-Authenticate": { + "schema": { + "type": "string" + }, + "description": "Bearer challenge (RFC 6750 §3). It carries error=\"invalid_token\" when a presented bearer credential was refused, and resource_metadata naming the OAuth protected-resource metadata when the server has an OIDC resource identity." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "403": { + "description": "Forbidden", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "404": { + "description": "not_found when no graph `graph_id` is served or the caller may not see it, or when no route serves the path", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "405": { + "description": "method_not_allowed when the path is served, but not with this method", + "headers": { + "Allow": { + "schema": { + "type": "string" + }, + "description": "The methods the path serves." + }, + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "412": { + "description": "actor_mismatch when Omnigraph-If-Actor names another actor than the credential resolves to", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "429": { + "description": "too_many_requests when server observer, ingress or operation admission capacity is exhausted; honor Retry-After", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + }, + "Retry-After": { + "schema": { + "type": "string" + }, + "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "500": { + "description": "Internal failure (code `internal`) before a successful response", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + }, + "503": { + "description": "graph_unavailable for a known graph that is not served, or service_unavailable when server operation admission is closed; honor Retry-After, and reconcile any earlier write before retrying", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + }, + "Retry-After": { + "schema": { + "type": "string" + }, + "description": "Minimum delay before a caller-bounded retry of this refused request; not a scheduled server retry." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorOutput" + } + } + } + } + }, + "security": [ + { + "bearer_token": [] + } + ] + } + }, + "/healthz": { + "get": { + "tags": [ + "health" + ], + "summary": "Liveness probe.", + "description": "Returns server status and version. Unauthenticated; safe to call from any\ncaller. Use this to confirm the server is reachable before invoking other\nendpoints.", + "operationId": "health", + "responses": { + "200": { + "description": "Server is healthy", + "headers": { + "Cache-Control": { + "schema": { + "type": "string", + "enum": [ + "no-store" + ] + } + }, + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HealthOutput" + } + } + } + } + } + }, + "head": { + "tags": [ + "health" + ], + "summary": "Liveness probe.", + "description": "Returns server status and version. Unauthenticated; safe to call from any\ncaller. Use this to confirm the server is reachable before invoking other\nendpoints.", + "operationId": "health_head", + "responses": { + "200": { + "description": "Server is healthy", + "headers": { + "Cache-Control": { + "schema": { + "type": "string", + "enum": [ + "no-store" + ] + } + }, + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + } + } + } + } + }, + "/readyz": { + "get": { + "tags": [ + "health" + ], + "summary": "Readiness witness (RFC 0049).", + "description": "Unauthenticated, and therefore minimal: it reports whether this replica\nis serving or draining, the applied `config_digest` it booted from, the\nledger revision and CAS it read, and registry/ready/loading/blocked counts. Graph\nids stay behind authenticated catalog endpoints. Partial availability is\nready but degraded after loading finishes; loading, no available graph or\nshutdown returns 503. A valid\nempty registry is ready. `/healthz` reports liveness independently.", + "operationId": "readiness", + "responses": { + "200": { + "description": "Serving", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ReadinessOutput" + } + } + } + }, + "503": { + "description": "Graphs loading, no graph available or stopping", + "headers": { + "Omnigraph-Http-Api": { + "schema": { + "type": "string", + "enum": [ + "0.14" + ] + }, + "description": "Supported OmniGraph HTTP API contract." + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ReadinessOutput" + } + } + } + } + } + } + } + }, + "components": { + "schemas": { + "AccessAction": { + "type": "string", + "description": "A graph action a policy decides, as the access route reports it.", + "enum": [ + "read", + "export", + "change", + "schema_apply", + "branch_create", + "branch_delete", + "branch_merge", + "commit_publish", + "commit_drop", + "invoke_query" + ] + }, + "AccessOutput": { + "type": "object", + "description": "What the caller's own credential may do on one graph, by the graph's\ninstalled policy. It names actions, never the rules that allow them, and\nsays nothing about any other actor. This closed vocabulary omits optimize;\nmaintenance status checks that permission separately on main.", + "required": [ + "graph_id", + "graph_wide", + "actions" + ], + "properties": { + "actions": { + "type": "array", + "items": { + "$ref": "#/components/schemas/AccessAction" + }, + "description": "With `branch`, the actions allowed on it, including `branch_delete`,\n`commit_publish`, `commit_drop` and `schema_apply` with it as their\ntarget. With `target_branch` as well, also `branch_create` and\n`branch_merge` from `branch` into it. Empty without `branch`." + }, + "actor_id": { + "type": [ + "string", + "null" + ], + "description": "The actor the credential resolves to. Absent only on a server that runs\nwithout authentication." + }, + "branch": { + "type": [ + "string", + "null" + ], + "description": "The branch asked about, when one was." + }, + "graph_id": { + "type": "string" + }, + "graph_wide": { + "type": "array", + "items": { + "$ref": "#/components/schemas/AccessAction" + }, + "description": "Actions allowed with no branch named: the graph-scoped ones, and the\nbranch-scoped ones a single rule allows on every branch (a rule with no\nbranch scope, `branch_scope: any` or `target_branch_scope: any`).\nRules that cover every branch only together, one for protected\nbranches and one for the rest, allow each named branch but leave the\naction out of this list; read `actions` for a branch." + }, + "target_branch": { + "type": [ + "string", + "null" + ], + "description": "The target branch asked about, when one was." + } + } + }, + "Applicability": { "type": "string", "description": "Whether a suggested source edit can be applied mechanically.", "enum": [ @@ -14060,885 +15058,1599 @@ } } }, - "DiagnosticOutput": { + "DiagnosticOutput": { + "type": "object", + "description": "The diagnostics contract for a refused query (RFC 0047): a stable code\n(`Q…` parse, `T…` typecheck); where the failure is, as a source position\nor as the stage and expression when it is post-parse; what was expected or\nviolated; and one concrete fix, absent when `expected` names the decision.\nRides `ErrorOutput.diagnostic` as an additive detail because\n[`ErrorCode`] is a closed compatibility contract.", + "required": [ + "code", + "expected" + ], + "properties": { + "code": { + "type": "string" + }, + "expected": { + "type": "string" + }, + "expression": { + "type": [ + "string", + "null" + ] + }, + "fix": { + "type": [ + "string", + "null" + ] + }, + "position": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/PositionOutput" + } + ] + }, + "stage": { + "type": [ + "string", + "null" + ] + }, + "suggestion": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/SuggestionOutput" + } + ] + } + } + }, + "DiffOutput": { + "type": "object", + "description": "One bounded page of a graph diff: the net entity changes between two\nresolved commits, in the commit diff's entity shape and order. There is no\ncause: a range has no single author.", + "required": [ + "from_commit_id", + "to_commit_id", + "changes" + ], + "properties": { + "base_commit_id": { + "type": [ + "string", + "null" + ], + "description": "The merge base, present only for a branch diff." + }, + "changes": { + "type": "array", + "items": { + "$ref": "#/components/schemas/EntityChangeOutput" + } + }, + "from_commit_id": { + "type": "string", + "description": "The commit the `from` side resolved to; for a branch diff, the merge\nbase." + }, + "next_page_token": { + "type": [ + "string", + "null" + ], + "description": "Continue THIS bounded response; later pages answer for the same two\ncommits. Absent on the final page." + }, + "to_commit_id": { + "type": "string", + "description": "The commit the `to` side resolved to; for a branch diff, the source\nhead." + } + } + }, + "DispositionAction": { + "type": "string", + "enum": [ + "fix", + "refresh", + "retry", + "recover", + "reconcile" + ] + }, + "DispositionEffects": { + "type": "string", + "enum": [ + "none", + "partial", + "unknown" + ] + }, + "DispositionOutput": { + "type": "object", + "description": "What a refused or failed request did and what its caller should do next.\nA server states it on every error response, computed where that knowledge\nis; a client that lost the response knows less, and says so.", + "required": [ + "effects", + "action" + ], + "properties": { + "action": { + "$ref": "#/components/schemas/DispositionAction" + }, + "effects": { + "$ref": "#/components/schemas/DispositionEffects" + } + } + }, + "EntityChangeOutput": { + "type": "object", + "description": "One entity change. Cause is stated once on the enclosing block, never here.\nAn insert carries only `after`, an update exact `before` and `after`, a\ndelete only `before`.", + "required": [ + "kind", + "type", + "id", + "op" + ], + "properties": { + "after": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/ChangeImageOutput" + } + ] + }, + "before": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/ChangeImageOutput" + } + ] + }, + "id": { + "type": "string" + }, + "kind": { + "$ref": "#/components/schemas/EntityKind" + }, + "op": { + "$ref": "#/components/schemas/ChangeOp" + }, + "type": { + "$ref": "#/components/schemas/ChangeTypeOutput" + } + } + }, + "EntityKind": { + "type": "string", + "description": "Logical graph entity namespace.", + "enum": [ + "node", + "edge" + ] + }, + "ErrorCode": { + "type": "string", + "description": "The registry of error codes. A server sends each code under the HTTP statuses named with it; a code without one never appears in an HTTP response.\n\n- `unauthorized` (401): The request carries no credential the server accepts: a bearer token is missing, unknown or expired. Supply one the server is configured to verify.\n- `forbidden` (403): The credential's actor may not do this: policy denies the action, or the action needs a policy permit the server has not been given.\n- `bad_request` (400): The request is malformed or invalid: a body, query string, path parameter or header that does not decode, a query the compiler refuses (see `diagnostic`), or a value refused before any effect.\n- `api_contract_mismatch` (400): The request lacks exactly one `Omnigraph-Http-Api` header naming the contract this server speaks. Authentication and this check precede graph access and effects.\n- `actor_mismatch` (412): The request's `Omnigraph-If-Actor` header names another actor than its credential resolves to. Nothing ran.\n- `outcome_unknown` (500): A write whose completion the server could not settle: it may or may not have published. Read the target before writing again; never replay the write blindly. An MCP write call that was cancelled or passed its deadline answers it too.\n- `precondition_failed` (412): A request precondition no longer holds: the branch head moved past `Omnigraph-If-Graph-Commit` (see `precondition_failure`), or a Blob's `If-Match` names a value the cell no longer holds (a write's `blob_precondition_failure` names the current one). Nothing was written.\n- `recovery_required` (503): An overlapping durable recovery intent must be resolved first; see `recovery_required`.\n- `external_blob_source` (424): An admitted external Blob source could not be probed or read; see `external_blob_source`.\n- `change_feed_gap` (410): Retained history can no longer reconstruct a change continuation; see `change_feed_gap`. Resume from a new baseline.\n- `not_found` (404): The graph, branch, commit, query, entity or route the request names does not exist, or the caller may not learn that it does.\n- `method_not_allowed` (405): The request path is served, but not with the request's method.\n- `request_timeout` (408): The request body did not arrive before its deadline; nothing was admitted.\n- `conflict` (409): The request conflicts with the graph's current state: a concurrent change, an existing id, a merge conflict, an index that must be built first, or a commit that cannot be diffed. The details say which.\n- `branch_start_refusal` (409, 410): `branch create` could not start the branch at the commit `from_commit` named, and nothing was created; see `branch_start_refusal`, whose `not_retained` reason answers 410.\n- `unattached_cannot_publish` (409): The commit is an unattached snapshot, a kind that never publishes. Distinct from `conflict`, the authority-moved refusal of a publishable proposal.\n- `payload_too_large` (413): The request body, or the work it asks for, exceeds a bounded limit; `resource_limit` names the limit when it is the operation's. Split or reduce the request.\n- `unsupported_media_type` (415): The request body's content type is not one the route accepts.\n- `range_not_satisfiable` (416): The requested Blob byte range lies outside the value; see `blob_range`.\n- `too_many_requests` (429): Admission capacity is exhausted: an actor's, the server's, or the MCP endpoint's concurrent tool calls. Nothing started; retry after `Retry-After`.\n- `service_unavailable` (503): Operation admission is closed, or a deployment observation raced an owner change. Nothing started; retry after `Retry-After`, reconciling any earlier write first.\n- `graph_unavailable` (503): A known graph is not being served. The request did not start, so it may be retried once the graph is served again.\n- `internal` (500): The server failed. An error before any write took no effect; see the server log.\n- `cancelled` (no HTTP status): MCP only: the client cancelled a read tool call. A cancelled write answers `outcome_unknown`.\n- `deadline_exceeded` (no HTTP status): MCP only: a read tool call passed its deadline. A write that does answers `outcome_unknown`.\n- `result_too_large` (no HTTP status): MCP only: a tool result exceeds the result limit. Narrow the read.", + "enum": [ + "unauthorized", + "forbidden", + "bad_request", + "api_contract_mismatch", + "actor_mismatch", + "outcome_unknown", + "precondition_failed", + "recovery_required", + "external_blob_source", + "change_feed_gap", + "not_found", + "method_not_allowed", + "request_timeout", + "conflict", + "branch_start_refusal", + "unattached_cannot_publish", + "payload_too_large", + "unsupported_media_type", + "range_not_satisfiable", + "too_many_requests", + "service_unavailable", + "graph_unavailable", + "internal", + "cancelled", + "deadline_exceeded", + "result_too_large" + ] + }, + "ErrorOutput": { "type": "object", - "description": "The diagnostics contract for a refused query (RFC 0047): a stable code\n(`Q…` parse, `T…` typecheck); where the failure is, as a source position\nor as the stage and expression when it is post-parse; what was expected or\nviolated; and one concrete fix, absent when `expected` names the decision.\nRides `ErrorOutput.diagnostic` as an additive detail because\n[`ErrorCode`] is a closed compatibility contract.", "required": [ + "error", "code", - "expected" + "disposition" ], "properties": { + "blob_precondition_failure": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/BlobPreconditionFailureOutput", + "description": "Set with HTTP 412 when a Blob write's `If-Match` did not hold for the\ncell. The `ETag` response header repeats `current_etag` when present." + } + ] + }, + "blob_range": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/BlobRangeOutput", + "description": "Set with HTTP 416 for a valid but unsatisfiable managed Blob byte range.\n`start..end` is half-open and `length` is the selected Blob length." + } + ] + }, + "branch_start_refusal": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/BranchStartRefusalOutput", + "description": "Set with HTTP 409, or 410 for `not_retained`, when `branch create`\ncannot start a branch at the commit `from_commit` named." + } + ] + }, + "change_diff_refusal": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/ChangeDiffRefusalOutput", + "description": "Set with HTTP 409 when a commit entity diff is refused (parentless\ncommit or an unprovable schema boundary)." + } + ] + }, + "change_feed_gap": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/ChangeFeedGapOutput", + "description": "Set with HTTP 410 when retained history can no longer reconstruct a\nchange continuation. Recover via the baseline handshake." + } + ] + }, "code": { - "type": "string" + "$ref": "#/components/schemas/ErrorCode", + "description": "The failure's registry code. Servers set it on every error response; it\nis absent only from a body built outside a server." }, - "expected": { + "diagnostic": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/DiagnosticOutput", + "description": "Set for a refused query: the diagnostics contract's code, position or\nstage, expectation and fix. `error` keeps the one-line rendering." + } + ] + }, + "disposition": { + "$ref": "#/components/schemas/DispositionOutput", + "description": "What the request did and what to do next. Servers set it on every error\nresponse; it is absent only from a body built outside a server." + }, + "error": { "type": "string" }, - "expression": { - "type": [ - "string", - "null" + "external_blob_source": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/ExternalBlobSourceOutput", + "description": "Set with HTTP 424 and `external_blob_source` when an external Blob URI\npassed admission policy but its source could not be probed or read." + } ] }, - "fix": { - "type": [ - "string", - "null" + "full_text_index_rebuild_required": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/FullTextIndexRebuildRequiredOutput", + "description": "Set with HTTP 409 when a selected full-text index requires an explicit\nrebuild before search can succeed. Unlike a write-authority conflict,\nthis condition is not cleared by retrying. This additive discriminator\npreserves the closed [`ErrorCode`] contract." + } ] }, - "position": { + "full_text_index_required": { "oneOf": [ { "type": "null" }, { - "$ref": "#/components/schemas/PositionOutput" + "$ref": "#/components/schemas/FullTextIndexRequiredOutput", + "description": "Set with HTTP 409 when a full-text call reads a declared index that\nhas no segment at the query's snapshot. Building the index clears it;\nretrying does not." } ] }, - "stage": { - "type": [ - "string", - "null" + "key_conflict": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/KeyConflictOutput", + "description": "Set when a strict keyed insert found an existing or concurrently\ninserted logical id. The caller may choose a different id; replaying\nthe same strict operation will not convert it into an upsert." + } ] }, - "suggestion": { + "merge_conflicts": { + "type": "array", + "items": { + "$ref": "#/components/schemas/MergeConflictOutput" + } + }, + "precondition_failure": { "oneOf": [ { "type": "null" }, { - "$ref": "#/components/schemas/SuggestionOutput" + "$ref": "#/components/schemas/PreconditionFailureOutput", + "description": "Set with HTTP 412 and `precondition_failed` when a mutation's\ngraph-commit precondition failed." + } + ] + }, + "published_dataset_version_conflict": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/PublishedDatasetVersionConflictOutput", + "description": "Set when the conflict is a publisher CAS rejection. The caller's\npre-write view named the expected published dataset version, but the\ngraph manifest now publishes the actual version. Refresh and retry." + } + ] + }, + "read_set_conflict": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/ReadSetConflictOutput", + "description": "Set when a prepared write's logical authority changed before effects." + } + ] + }, + "recovery_required": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/RecoveryRequiredOutput", + "description": "Set when an overlapping durable recovery intent must be resolved before\nretry. Its dataset effects may or may not have started." + } + ] + }, + "resource_limit": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/ResourceLimitOutput", + "description": "Set when the request must be split into smaller graph commits. The\nrejected attempt has no durable effect." } ] } } }, - "DiffOutput": { + "ExportRequest": { "type": "object", - "description": "One bounded page of a graph diff: the net entity changes between two\nresolved commits, in the commit diff's entity shape and order. There is no\ncause: a range has no single author.", - "required": [ - "from_commit_id", - "to_commit_id", - "changes" - ], "properties": { - "base_commit_id": { + "branch": { "type": [ "string", "null" ], - "description": "The merge base, present only for a branch diff." + "description": "Branch to export. Defaults to `main`." }, - "changes": { + "type_names": { "type": "array", "items": { - "$ref": "#/components/schemas/EntityChangeOutput" - } - }, - "from_commit_id": { + "type": "string" + }, + "description": "Restrict the export to these node/edge type names. Empty exports all types." + } + }, + "additionalProperties": false + }, + "ExternalBlobSourceOutput": { + "type": "object", + "description": "Structured details for an allowed external Blob source that could not be\nprobed or read, sent with [`ErrorCode::ExternalBlobSource`].", + "required": [ + "uri", + "reason" + ], + "properties": { + "reason": { "type": "string", - "description": "The commit the `from` side resolved to; for a branch diff, the merge\nbase." + "description": "Source-side failure diagnosis. Clients should branch on the presence of\n`external_blob_source`, not parse this human-readable text." }, - "next_page_token": { - "type": [ - "string", - "null" - ], - "description": "Continue THIS bounded response; later pages answer for the same two\ncommits. Absent on the final page." + "uri": { + "type": "string", + "description": "Normalized, credential-free URI spelling (or a redacted placeholder)." + } + } + }, + "FullTextIndexRebuildRequiredOutput": { + "type": "object", + "description": "A selected full-text index cannot safely serve the current analyzer (HTTP 409).\nThis is not a retryable write conflict: an operator must rebuild the live\nbranch's indexes. Historical snapshots stay unchanged; branch old content\nand rebuild that branch to search it.", + "required": [ + "index", + "reason" + ], + "properties": { + "index": { + "type": "string" }, - "to_commit_id": { + "reason": { + "type": "string", + "description": "Human-readable diagnosis; branch on the enclosing detail's presence,\nnot this text, to distinguish the operator-action-required condition." + } + } + }, + "FullTextIndexRequiredOutput": { + "type": "object", + "description": "A full-text call read a declared full-text index that has no segment at\nthe query's snapshot (HTTP 409), so no segment carries its analyzer. This\nis not a retryable conflict: an operator builds the branch's index\n(`omnigraph optimize` on main, `omnigraph rebuild-full-text-indexes\n--branch` elsewhere). Historical snapshots stay unchanged.", + "required": [ + "index" + ], + "properties": { + "index": { "type": "string", - "description": "The commit the `to` side resolved to; for a branch diff, the source\nhead." + "description": "The index's property as `Type.property`." } } }, - "DispositionAction": { + "GraphAvailability": { "type": "string", "enum": [ - "fix", - "refresh", - "retry", - "recover", - "reconcile" + "loading", + "ready", + "transitioning", + "blocked", + "stopping" ] }, - "DispositionEffects": { + "GraphAvailabilityAction": { "type": "string", "enum": [ "none", - "partial", - "unknown" + "wait_for_startup", + "wait_for_transition", + "apply_correction_or_restart", + "wait_for_restart" ] }, - "DispositionOutput": { + "GraphBatchDeclarationOutput": { "type": "object", - "description": "What a refused or failed request did and what its caller should do next.\nA server states it on every error response, computed where that knowledge\nis; a client that lost the response knows less, and says so.", + "description": "One logical declaration touched by a graph-batch load.\n\nThis deliberately carries the accepted-schema name, not a backing dataset\nselector, path, or Lance identity.", "required": [ - "effects", - "action" + "name", + "entities_loaded" ], "properties": { - "action": { - "$ref": "#/components/schemas/DispositionAction" + "entities_loaded": { + "type": "integer", + "minimum": 0 }, - "effects": { - "$ref": "#/components/schemas/DispositionEffects" + "name": { + "type": "string" } } }, - "EntityChangeOutput": { + "GraphBatchLoadOutput": { "type": "object", - "description": "One entity change. Cause is stated once on the enclosing block, never here.\nAn insert carries only `after`, an update exact `before` and `after`, a\ndelete only `before`.", + "description": "Terminal result for the raw graph-level NDJSON load surface.", "required": [ - "kind", - "type", - "id", - "op" + "branch", + "branch_created", + "mode", + "nodes", + "edges", + "total_entities", + "embedding_generation" ], "properties": { - "after": { + "actor_id": { + "type": [ + "string", + "null" + ] + }, + "base_branch": { + "type": [ + "string", + "null" + ], + "description": "Base branch a fork was requested from, even when the target already\nexisted. `null` when the request omitted `from`." + }, + "branch": { + "type": "string" + }, + "branch_created": { + "type": "boolean" + }, + "commit": { "oneOf": [ { "type": "null" }, { - "$ref": "#/components/schemas/ChangeImageOutput" + "$ref": "#/components/schemas/CommitOutput" } ] }, - "before": { + "edges": { + "type": "array", + "items": { + "$ref": "#/components/schemas/GraphBatchDeclarationOutput" + }, + "description": "Logical edge declarations touched by this batch, sorted by name." + }, + "embedding_generation": { "oneOf": [ { "type": "null" }, { - "$ref": "#/components/schemas/ChangeImageOutput" + "$ref": "#/components/schemas/LoadEmbeddingGeneration", + "description": "`unsupported` when a loaded node type declares `@embed`, including\nwhen all vectors were supplied; `null` otherwise." } ] }, - "id": { - "type": "string" - }, - "kind": { - "$ref": "#/components/schemas/EntityKind" + "mode": { + "$ref": "#/components/schemas/LoadMode" }, - "op": { - "$ref": "#/components/schemas/ChangeOp" + "nodes": { + "type": "array", + "items": { + "$ref": "#/components/schemas/GraphBatchDeclarationOutput" + }, + "description": "Logical node declarations touched by this batch, sorted by name." }, - "type": { - "$ref": "#/components/schemas/ChangeTypeOutput" + "total_entities": { + "type": "integer", + "minimum": 0 } } }, - "EntityKind": { - "type": "string", - "description": "Logical graph entity namespace.", - "enum": [ - "node", - "edge" - ] + "GraphDiscoveryEntry": { + "type": "object", + "description": "A graph's existence, without storage, schema, data, or serving metadata.", + "required": [ + "graph_id", + "display_name" + ], + "properties": { + "display_name": { + "type": "string", + "description": "Currently the graph identifier; no separate display name is configured." + }, + "graph_id": { + "type": "string" + } + }, + "additionalProperties": false }, - "ErrorCode": { - "type": "string", - "description": "The registry of error codes. A server sends each code under the HTTP statuses named with it; a code without one never appears in an HTTP response.\n\n- `unauthorized` (401): The request carries no credential the server accepts: a bearer token is missing, unknown or expired. Supply one the server is configured to verify.\n- `forbidden` (403): The credential's actor may not do this: policy denies the action, or the action needs a policy permit the server has not been given.\n- `bad_request` (400): The request is malformed or invalid: a body, query string, path parameter or header that does not decode, a query the compiler refuses (see `diagnostic`), or a value refused before any effect.\n- `api_contract_mismatch` (400): The request lacks exactly one `Omnigraph-Http-Api` header naming the contract this server speaks. Authentication and this check precede graph access and effects.\n- `actor_mismatch` (412): The request's `Omnigraph-If-Actor` header names another actor than its credential resolves to. Nothing ran.\n- `outcome_unknown` (500): A write whose completion the server could not settle: it may or may not have published. Read the target before writing again; never replay the write blindly. An MCP write call that was cancelled or passed its deadline answers it too.\n- `precondition_failed` (412): A request precondition no longer holds: the branch head moved past `Omnigraph-If-Graph-Commit` (see `precondition_failure`), or a Blob's `If-Match` names a value the cell no longer holds (a write's `blob_precondition_failure` names the current one). Nothing was written.\n- `recovery_required` (503): An overlapping durable recovery intent must be resolved first; see `recovery_required`.\n- `external_blob_source` (424): An admitted external Blob source could not be probed or read; see `external_blob_source`.\n- `change_feed_gap` (410): Retained history can no longer reconstruct a change continuation; see `change_feed_gap`. Resume from a new baseline.\n- `not_found` (404): The graph, branch, commit, query, entity or route the request names does not exist, or the caller may not learn that it does.\n- `method_not_allowed` (405): The request path is served, but not with the request's method.\n- `request_timeout` (408): The request body did not arrive before its deadline; nothing was admitted.\n- `conflict` (409): The request conflicts with the graph's current state: a concurrent change, an existing id, a merge conflict, an index that must be built first, or a commit that cannot be diffed. The details say which.\n- `branch_start_refusal` (409, 410): `branch create` could not start the branch at the commit `from_commit` named, and nothing was created; see `branch_start_refusal`, whose `not_retained` reason answers 410.\n- `unattached_cannot_publish` (409): The commit is an unattached snapshot, a kind that never publishes. Distinct from `conflict`, the authority-moved refusal of a publishable proposal.\n- `payload_too_large` (413): The request body, or the work it asks for, exceeds a bounded limit; `resource_limit` names the limit when it is the operation's. Split or reduce the request.\n- `unsupported_media_type` (415): The request body's content type is not one the route accepts.\n- `range_not_satisfiable` (416): The requested Blob byte range lies outside the value; see `blob_range`.\n- `too_many_requests` (429): Admission capacity is exhausted: an actor's, the server's, or the MCP endpoint's concurrent tool calls. Nothing started; retry after `Retry-After`.\n- `service_unavailable` (503): Operation admission is closed, or a deployment observation raced an owner change. Nothing started; retry after `Retry-After`, reconciling any earlier write first.\n- `graph_unavailable` (503): A known graph is not being served. The request did not start, so it may be retried once the graph is served again.\n- `internal` (500): The server failed. An error before any write took no effect; see the server log.\n- `cancelled` (no HTTP status): MCP only: the client cancelled a read tool call. A cancelled write answers `outcome_unknown`.\n- `deadline_exceeded` (no HTTP status): MCP only: a read tool call passed its deadline. A write that does answers `outcome_unknown`.\n- `result_too_large` (no HTTP status): MCP only: a tool result exceeds the result limit. Narrow the read.", - "enum": [ - "unauthorized", - "forbidden", - "bad_request", - "api_contract_mismatch", - "actor_mismatch", - "outcome_unknown", - "precondition_failed", - "recovery_required", - "external_blob_source", - "change_feed_gap", - "not_found", - "method_not_allowed", - "request_timeout", - "conflict", - "branch_start_refusal", - "unattached_cannot_publish", - "payload_too_large", - "unsupported_media_type", - "range_not_satisfiable", - "too_many_requests", - "service_unavailable", - "graph_unavailable", - "internal", - "cancelled", - "deadline_exceeded", - "result_too_large" - ] + "GraphDiscoveryOutput": { + "type": "object", + "description": "Authenticated minimal inventory from `GET /graphs/discovery`.", + "required": [ + "graphs" + ], + "properties": { + "graphs": { + "type": "array", + "items": { + "$ref": "#/components/schemas/GraphDiscoveryEntry" + } + } + }, + "additionalProperties": false }, - "ErrorOutput": { + "GraphInfo": { "type": "object", + "description": "One entry in the response from `GET /graphs`. Cluster operators\nconsume this list to discover which graphs the server is currently\nserving. This legacy metadata includes the storage `uri`; identity-only\nexistence discovery uses [`GraphDiscoveryEntry`] instead.", "required": [ - "error", - "code", - "disposition" + "graph_id", + "uri", + "state", + "read_available", + "write_available", + "action" ], "properties": { - "blob_precondition_failure": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/components/schemas/BlobPreconditionFailureOutput", - "description": "Set with HTTP 412 when a Blob write's `If-Match` did not hold for the\ncell. The `ETag` response header repeats `current_etag` when present." - } - ] - }, - "blob_range": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/components/schemas/BlobRangeOutput", - "description": "Set with HTTP 416 for a valid but unsatisfiable managed Blob byte range.\n`start..end` is half-open and `length` is the selected Blob length." - } - ] - }, - "branch_start_refusal": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/components/schemas/BranchStartRefusalOutput", - "description": "Set with HTTP 409, or 410 for `not_retained`, when `branch create`\ncannot start a branch at the commit `from_commit` named." - } - ] - }, - "change_diff_refusal": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/components/schemas/ChangeDiffRefusalOutput", - "description": "Set with HTTP 409 when a commit entity diff is refused (parentless\ncommit or an unprovable schema boundary)." - } - ] - }, - "change_feed_gap": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/components/schemas/ChangeFeedGapOutput", - "description": "Set with HTTP 410 when retained history can no longer reconstruct a\nchange continuation. Recover via the baseline handshake." - } - ] - }, - "code": { - "$ref": "#/components/schemas/ErrorCode", - "description": "The failure's registry code. Servers set it on every error response; it\nis absent only from a body built outside a server." + "action": { + "$ref": "#/components/schemas/GraphAvailabilityAction" }, - "diagnostic": { + "failure": { "oneOf": [ { "type": "null" }, { - "$ref": "#/components/schemas/DiagnosticOutput", - "description": "Set for a refused query: the diagnostics contract's code, position or\nstage, expectation and fix. `error` keeps the one-line rendering." + "$ref": "#/components/schemas/GraphStartupFailure" } ] }, - "disposition": { - "$ref": "#/components/schemas/DispositionOutput", - "description": "What the request did and what to do next. Servers set it on every error\nresponse; it is absent only from a body built outside a server." + "graph_id": { + "type": "string" }, - "error": { + "read_available": { + "type": "boolean", + "description": "Runtime availability; actor policy still gates every operation." + }, + "state": { + "$ref": "#/components/schemas/GraphAvailability" + }, + "uri": { "type": "string" }, - "external_blob_source": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/components/schemas/ExternalBlobSourceOutput", - "description": "Set with HTTP 424 and `external_blob_source` when an external Blob URI\npassed admission policy but its source could not be probed or read." - } - ] + "write_available": { + "type": "boolean" + } + } + }, + "GraphListOutput": { + "type": "object", + "description": "Response from `GET /graphs`. Lists every graph registered with the\nserver in alphabetical order by `graph_id` (sorted server-side so\nclients get deterministic output across requests).", + "required": [ + "graphs" + ], + "properties": { + "graphs": { + "type": "array", + "items": { + "$ref": "#/components/schemas/GraphInfo" + } + } + } + }, + "GraphStartupFailure": { + "type": "string", + "description": "Bounded startup classification, without storage paths or error text.", + "enum": [ + "invalid_configuration", + "invalid_policy", + "invalid_external_blob_policy", + "open_failed", + "invalid_stored_queries" + ] + }, + "HealthOutput": { + "type": "object", + "required": [ + "status", + "version", + "internal_schema_version" + ], + "properties": { + "internal_schema_version": { + "type": "integer", + "format": "int32", + "description": "The internal-schema (storage-format) version this binary writes, the\nhighest it serves. Normal open serves every version from the binary's\nlowest supported one up to this one; a graph below that range is\nrefused until it is upgraded offline or rebuilt, and a graph above it\nneeds a newer binary.", + "minimum": 0 }, - "full_text_index_rebuild_required": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/components/schemas/FullTextIndexRebuildRequiredOutput", - "description": "Set with HTTP 409 when a selected full-text index requires an explicit\nrebuild before search can succeed. Unlike a write-authority conflict,\nthis condition is not cleared by retrying. This additive discriminator\npreserves the closed [`ErrorCode`] contract." - } + "source_version": { + "type": [ + "string", + "null" ] }, - "full_text_index_required": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/components/schemas/FullTextIndexRequiredOutput", - "description": "Set with HTTP 409 when a full-text call reads a declared index that\nhas no segment at the query's snapshot. Building the index clears it;\nretrying does not." - } + "status": { + "type": "string" + }, + "version": { + "type": "string" + } + } + }, + "IngestOutput": { + "type": "object", + "required": [ + "uri", + "branch", + "branch_created", + "mode", + "nodes", + "edges", + "total_entities", + "embedding_generation" + ], + "properties": { + "actor_id": { + "type": [ + "string", + "null" ] }, - "key_conflict": { + "base_branch": { + "type": [ + "string", + "null" + ], + "description": "Base branch a fork was requested from (the request's `from`), echoed\neven when the branch already existed. `null` when `from` was absent." + }, + "branch": { + "type": "string" + }, + "branch_created": { + "type": "boolean" + }, + "commit": { "oneOf": [ { "type": "null" }, { - "$ref": "#/components/schemas/KeyConflictOutput", - "description": "Set when a strict keyed insert found an existing or concurrently\ninserted logical id. The caller may choose a different id; replaying\nthe same strict operation will not convert it into an upsert." + "$ref": "#/components/schemas/CommitOutput" } ] }, - "merge_conflicts": { + "edges": { "type": "array", "items": { - "$ref": "#/components/schemas/MergeConflictOutput" - } + "$ref": "#/components/schemas/GraphBatchDeclarationOutput" + }, + "description": "Logical edge declarations touched by this load, sorted by name." }, - "precondition_failure": { + "embedding_generation": { "oneOf": [ { "type": "null" }, { - "$ref": "#/components/schemas/PreconditionFailureOutput", - "description": "Set with HTTP 412 and `precondition_failed` when a mutation's\ngraph-commit precondition failed." + "$ref": "#/components/schemas/LoadEmbeddingGeneration", + "description": "`unsupported` when a loaded node type declares `@embed`, including\nwhen all vectors were supplied; `null` otherwise." } ] }, - "published_dataset_version_conflict": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/components/schemas/PublishedDatasetVersionConflictOutput", - "description": "Set when the conflict is a publisher CAS rejection. The caller's\npre-write view named the expected published dataset version, but the\ngraph manifest now publishes the actual version. Refresh and retry." - } - ] + "mode": { + "$ref": "#/components/schemas/LoadMode" }, - "read_set_conflict": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/components/schemas/ReadSetConflictOutput", - "description": "Set when a prepared write's logical authority changed before effects." - } - ] + "nodes": { + "type": "array", + "items": { + "$ref": "#/components/schemas/GraphBatchDeclarationOutput" + }, + "description": "Logical node declarations touched by this load, sorted by name." }, - "recovery_required": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/components/schemas/RecoveryRequiredOutput", - "description": "Set when an overlapping durable recovery intent must be resolved before\nretry. Its dataset effects may or may not have started." - } - ] + "total_entities": { + "type": "integer", + "minimum": 0 }, - "resource_limit": { + "uri": { + "type": "string" + } + } + }, + "IngestRequest": { + "type": "object", + "required": [ + "data" + ], + "properties": { + "branch": { + "type": [ + "string", + "null" + ], + "description": "Target branch. Defaults to `main`. Without `from`, the branch must\nalready exist — a missing branch is a 404, never an implicit fork." + }, + "data": { + "type": "string", + "description": "NDJSON payload: one record per line, each shaped\n`{\"type\": \"\", \"data\": {...}}`.", + "example": "{\"type\": \"Person\", \"data\": {\"name\": \"Alice\", \"age\": 30}}\n{\"type\": \"Person\", \"data\": {\"name\": \"Bob\", \"age\": 25}}" + }, + "from": { + "type": [ + "string", + "null" + ], + "description": "Parent branch used to create `branch` if it does not exist. Branch\ncreation is opt-in by presence of this field; omit it to require an\nexisting branch." + }, + "mode": { "oneOf": [ { "type": "null" }, { - "$ref": "#/components/schemas/ResourceLimitOutput", - "description": "Set when the request must be split into smaller graph commits. The\nrejected attempt has no durable effect." + "$ref": "#/components/schemas/LoadMode", + "description": "How existing entities are handled. Defaults to `merge`." } ] } } }, - "ExportRequest": { + "InvokeStoredQueryOutput": { + "oneOf": [ + { + "$ref": "#/components/schemas/ReadOutput" + }, + { + "$ref": "#/components/schemas/ChangeOutput" + } + ], + "description": "Response for `POST /queries/{name}`: the read envelope for a stored\nread, or the mutation envelope for a stored mutation. Serialized\n**untagged**, so the wire shape is exactly [`ReadOutput`] or\n[`ChangeOutput`] — classification follows the stored query, not a\nwrapper field." + }, + "InvokeStoredQueryRequest": { "type": "object", + "description": "Body for `POST /queries/{name}` — invokes the server-side stored query\nnamed in the path. The query source and name come from the registry,\nnever the body; only the runtime inputs are supplied here.", "properties": { "branch": { "type": [ "string", "null" ], - "description": "Branch to export. Defaults to `main`." - }, - "type_names": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Restrict the export to these node/edge type names. Empty exports all types." + "description": "Branch to run against. Defaults to `main`; for a stored mutation the\nwrite targets this branch." + }, + "expect_mutation": { + "type": [ + "boolean", + "null" + ], + "description": "The kind the caller expects: `Some(false)` for\n`omnigraph query `, `Some(true)` for `omnigraph mutate `.\nWhen set and it disagrees with the stored query's actual kind, the\nserver rejects the call (400) so the verb asserts the kind. `None`\n(the default) skips the check — preserving older clients and aliases." + }, + "params": { + "description": "JSON object whose keys match the stored query's declared parameters." + }, + "snapshot": { + "type": [ + "string", + "null" + ], + "description": "Snapshot id to read from (read queries only — rejected for a stored\nmutation). Mutually exclusive with `branch`." } - }, - "additionalProperties": false + } }, - "ExternalBlobSourceOutput": { + "KeyConflictOutput": { "type": "object", - "description": "Structured details for an allowed external Blob source that could not be\nprobed or read, sent with [`ErrorCode::ExternalBlobSource`].", + "description": "A strict insert rejected because `entity_id` already names an entity in the\nselected node or edge type. The refused write changed nothing; an earlier\nstep of the same request, such as the branch a load forks, may already have\npublished, which the error's `disposition` states. Partial or ambiguous\nattempts surface `recovery_required` instead.", "required": [ - "uri", - "reason" + "entity_kind", + "type_name" ], "properties": { - "reason": { - "type": "string", - "description": "Source-side failure diagnosis. Clients should branch on the presence of\n`external_blob_source`, not parse this human-readable text." + "entity_id": { + "type": [ + "string", + "null" + ] }, - "uri": { - "type": "string", - "description": "Normalized, credential-free URI spelling (or a redacted placeholder)." + "entity_kind": { + "$ref": "#/components/schemas/EntityKind" + }, + "type_name": { + "type": "string" } } }, - "FullTextIndexRebuildRequiredOutput": { + "LoadEmbeddingGeneration": { + "type": "string", + "description": "Load capability for a batch touching a node type with declared `@embed`.", + "enum": [ + "unsupported" + ] + }, + "LoadExecutedOutput": { "type": "object", - "description": "A selected full-text index cannot safely serve the current analyzer (HTTP 409).\nThis is not a retryable write conflict: an operator must rebuild the live\nbranch's indexes. Historical snapshots stay unchanged; branch old content\nand rebuild that branch to search it.", + "description": "What a `load csv` statement's plan measured as it ran, beside the\nstatement's [`ChangeOutput`]: the source partitions and bytes it read, the\ncount its sink staged, and what each operator did. Absent for an append\nor merge load, which the keyed adapters run, and for every other change.", "required": [ - "index", - "reason" + "partitions", + "rows", + "source_bytes", + "operators" ], "properties": { - "index": { - "type": "string" + "operators": { + "type": "array", + "items": { + "$ref": "#/components/schemas/LoadOperatorExecutedOutput" + }, + "description": "One entry per node of the executed plan in post-order, the source\nfirst." }, - "reason": { - "type": "string", - "description": "Human-readable diagnosis; branch on the enclosing detail's presence,\nnot this text, to distinguish the operator-action-required condition." + "partitions": { + "type": "integer", + "format": "int32", + "description": "Source partitions the plan read.", + "minimum": 0 + }, + "rows": { + "type": "integer", + "format": "int64", + "description": "Entities the sink staged.", + "minimum": 0 + }, + "source_bytes": { + "type": "integer", + "format": "int64", + "description": "Bytes of the captured source.", + "minimum": 0 } } }, - "FullTextIndexRequiredOutput": { + "LoadMode": { + "type": "string", + "description": "Shadow enum for documenting [`LoadMode`] in the OpenAPI schema.", + "enum": [ + "overwrite", + "append", + "merge" + ] + }, + "LoadOperatorExecutedOutput": { "type": "object", - "description": "A full-text call read a declared full-text index that has no segment at\nthe query's snapshot (HTTP 409), so no segment carries its analyzer. This\nis not a retryable conflict: an operator builds the branch's index\n(`omnigraph optimize` on main, `omnigraph rebuild-full-text-indexes\n--branch` elsewhere). Historical snapshots stay unchanged.", + "description": "What one operator of an executed load plan did.", "required": [ - "index" + "id", + "node", + "operator", + "rows", + "elapsed_compute_ns", + "wall_ns" ], "properties": { - "index": { + "elapsed_compute_ns": { + "type": "integer", + "format": "int64", + "description": "Nanoseconds the operator was computing: the polls of its stream and of\nits workers, never the time it waited for its input or for storage.", + "minimum": 0 + }, + "id": { + "type": "integer", + "format": "int64", + "description": "The plan node's `id`, as the explain document numbers it.", + "minimum": 0 + }, + "node": { "type": "string", - "description": "The index's property as `Type.property`." + "description": "The plan node's kind: `CsvSource`, `Validate`, `FragmentWrite`." + }, + "operator": { + "type": "string", + "description": "The operator the node built." + }, + "partitions": { + "type": "integer", + "format": "int32", + "description": "The output partitions the operator ran, each its own stream: a\n`CsvSource`'s parsers, 1 for every other operator.", + "minimum": 0 + }, + "rows": { + "type": "integer", + "format": "int64", + "description": "What the operator emitted; the sink counts what it staged.", + "minimum": 0 + }, + "slices": { + "type": "array", + "items": { + "$ref": "#/components/schemas/LoadSliceExecutedOutput" + }, + "description": "The slices of the source a `CsvSource` cut and parsed, in slice order;\nabsent for every other operator." + }, + "wall_ns": { + "type": "integer", + "format": "int64", + "description": "Nanoseconds from the operator's first poll to its end, waits included;\n`0` where no end was recorded.", + "minimum": 0 } } }, - "GraphAvailability": { - "type": "string", - "enum": [ - "loading", - "ready", - "transitioning", - "blocked", - "stopping" - ] - }, - "GraphAvailabilityAction": { - "type": "string", - "enum": [ - "none", - "wait_for_startup", - "wait_for_transition", - "apply_correction_or_restart", - "wait_for_restart" - ] - }, - "GraphBatchDeclarationOutput": { + "LoadSliceExecutedOutput": { "type": "object", - "description": "One logical declaration touched by a graph-batch load.\n\nThis deliberately carries the accepted-schema name, not a backing dataset\nselector, path, or Lance identity.", + "description": "What one slice of a `CsvSource` measured: a byte range of the capture\nparsed whole by one partition.", "required": [ - "name", - "entities_loaded" + "slice", + "partition", + "rows", + "elapsed_compute_ns", + "start_ns", + "end_ns" ], "properties": { - "entities_loaded": { + "elapsed_compute_ns": { "type": "integer", + "format": "int64", + "description": "Nanoseconds the slice's parse and normalization took.", "minimum": 0 }, - "name": { - "type": "string" + "end_ns": { + "type": "integer", + "format": "int64", + "description": "Nanoseconds after the source's first poll at which the slice's last\nbatch was sent.", + "minimum": 0 + }, + "partition": { + "type": "integer", + "format": "int32", + "description": "The partition that claimed the slice; which one varies between runs.", + "minimum": 0 + }, + "read_ns": { + "type": "integer", + "format": "int64", + "description": "Nanoseconds the slice's partition waited for the source's bytes.", + "minimum": 0 + }, + "rows": { + "type": "integer", + "format": "int64", + "description": "The records the slice held.", + "minimum": 0 + }, + "slice": { + "type": "integer", + "format": "int32", + "description": "The slice's position in the source, from 0; a slice's rows are the\nsame on every run.", + "minimum": 0 + }, + "start_ns": { + "type": "integer", + "format": "int64", + "description": "Nanoseconds after the source's first poll at which the slice was\nclaimed.", + "minimum": 0 } } }, - "GraphBatchLoadOutput": { + "MaintenanceInvocation": { "type": "object", - "description": "Terminal result for the raw graph-level NDJSON load surface.", "required": [ - "branch", - "branch_created", - "mode", - "nodes", - "edges", - "total_entities", - "embedding_generation" + "request", + "actor", + "canonical_root", + "result_revision", + "schema_contract", + "input", + "publication", + "settlement" ], "properties": { - "actor_id": { - "type": [ - "string", - "null" - ] - }, - "base_branch": { - "type": [ - "string", - "null" - ], - "description": "Base branch a fork was requested from, even when the target already\nexisted. `null` when the request omitted `from`." - }, - "branch": { + "actor": { "type": "string" }, - "branch_created": { - "type": "boolean" + "canonical_root": { + "type": "string" }, - "commit": { + "input": { "oneOf": [ { "type": "null" }, { - "$ref": "#/components/schemas/CommitOutput" + "$ref": "#/components/schemas/OptimizeInput", + "description": "Accepted graph commit and source inventory, bound before physical work." } - ] - }, - "edges": { - "type": "array", - "items": { - "$ref": "#/components/schemas/GraphBatchDeclarationOutput" - }, - "description": "Logical edge declarations touched by this batch, sorted by name." + ] }, - "embedding_generation": { + "publication": { "oneOf": [ { "type": "null" }, { - "$ref": "#/components/schemas/LoadEmbeddingGeneration", - "description": "`unsupported` when a loaded node type declares `@embed`, including\nwhen all vectors were supplied; `null` otherwise." + "$ref": "#/components/schemas/OptimizePublication" } ] }, - "mode": { - "$ref": "#/components/schemas/LoadMode" - }, - "nodes": { - "type": "array", - "items": { - "$ref": "#/components/schemas/GraphBatchDeclarationOutput" - }, - "description": "Logical node declarations touched by this batch, sorted by name." + "request": { + "$ref": "#/components/schemas/MaintenanceRequest" }, - "total_entities": { + "result_revision": { "type": "integer", + "format": "int64", "minimum": 0 + }, + "schema_contract": { + "$ref": "#/components/schemas/MaintenanceSchemaContract", + "description": "Applied configuration contract at admission, not an execution-time head." + }, + "settlement": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/OptimizeSettlement" + } + ] } - } + }, + "additionalProperties": false }, - "GraphDiscoveryEntry": { + "MaintenanceKind": { + "type": "string", + "enum": [ + "optimize" + ] + }, + "MaintenanceLimits": { "type": "object", - "description": "A graph's existence, without storage, schema, data, or serving metadata.", + "description": "Positive source ceilings for one optimization; none may exceed the server's\nadvertised limits. They do not bound process memory, CPU or total I/O.", "required": [ - "graph_id", - "display_name" + "max_tables", + "max_fragments", + "max_rows", + "max_source_bytes", + "max_files" ], "properties": { - "display_name": { - "type": "string", - "description": "Currently the graph identifier; no separate display name is configured." + "max_files": { + "type": "integer", + "format": "int64", + "minimum": 1 }, - "graph_id": { - "type": "string" + "max_fragments": { + "type": "integer", + "format": "int64", + "minimum": 1 + }, + "max_rows": { + "type": "integer", + "format": "int64", + "minimum": 1 + }, + "max_source_bytes": { + "type": "integer", + "format": "int64", + "minimum": 1 + }, + "max_tables": { + "type": "integer", + "format": "int32", + "minimum": 1 } }, "additionalProperties": false }, - "GraphDiscoveryOutput": { + "MaintenanceLookup": { + "oneOf": [ + { + "type": "object", + "required": [ + "operation_id", + "graph", + "invocation", + "status" + ], + "properties": { + "graph": { + "type": "string" + }, + "invocation": { + "$ref": "#/components/schemas/MaintenanceInvocation" + }, + "operation_id": { + "type": "string" + }, + "status": { + "type": "string", + "enum": [ + "outstanding" + ] + } + } + }, + { + "type": "object", + "required": [ + "result", + "status" + ], + "properties": { + "result": { + "$ref": "#/components/schemas/MaintenanceResult" + }, + "status": { + "type": "string", + "enum": [ + "complete" + ] + } + } + }, + { + "type": "object", + "required": [ + "operation_id", + "status" + ], + "properties": { + "operation_id": { + "type": "string" + }, + "status": { + "type": "string", + "enum": [ + "result_expired" + ] + } + } + }, + { + "type": "object", + "required": [ + "operation_id", + "status" + ], + "properties": { + "operation_id": { + "type": "string" + }, + "status": { + "type": "string", + "enum": [ + "not_recorded" + ] + } + } + }, + { + "type": "object", + "required": [ + "operation_id", + "status" + ], + "properties": { + "operation_id": { + "type": "string" + }, + "status": { + "type": "string", + "enum": [ + "identity_mismatch" + ] + } + } + }, + { + "type": "object", + "required": [ + "operation_id", + "status" + ], + "properties": { + "operation_id": { + "type": "string" + }, + "status": { + "type": "string", + "enum": [ + "different_ledger" + ] + } + } + } + ] + }, + "MaintenanceOutput": { "type": "object", - "description": "Authenticated minimal inventory from `GET /graphs/discovery`.", "required": [ - "graphs" + "maintenance", + "in_progress" ], "properties": { - "graphs": { - "type": "array", - "items": { - "$ref": "#/components/schemas/GraphDiscoveryEntry" - } + "in_progress": { + "type": "boolean", + "description": "Current process owns this ID. Ownership alone is not durable acceptance;\nan outstanding receipt without an owner remains unresolved." + }, + "maintenance": { + "$ref": "#/components/schemas/MaintenanceLookup", + "description": "Native lookup tagged by `status`: outstanding, complete, result_expired,\nnot_recorded, identity_mismatch or different_ledger. Complete contains\n`result` with operation_id, graph, invocation and outcome: success with\nits exact report, or refused with code/message. Only success proves work\ncompleted; missing/expired evidence never permits replay." } }, - "additionalProperties": false + "example": { + "maintenance": { + "status": "not_recorded", + "operation_id": "01ARZ3NDEKTSV4RRFFQ69G5FAV:1:01ARZ3NDEKTSV4RRFFQ69G5FAW" + }, + "in_progress": false + } }, - "GraphInfo": { + "MaintenancePublication": { + "oneOf": [ + { + "type": "object", + "required": [ + "status" + ], + "properties": { + "status": { + "type": "string", + "enum": [ + "unpublished" + ] + } + } + }, + { + "type": "object", + "required": [ + "report", + "status" + ], + "properties": { + "report": { + "$ref": "#/components/schemas/OptimizeReport" + }, + "status": { + "type": "string", + "enum": [ + "published" + ] + } + } + } + ] + }, + "MaintenanceRequest": { "type": "object", - "description": "One entry in the response from `GET /graphs`. Cluster operators\nconsume this list to discover which graphs the server is currently\nserving. This legacy metadata includes the storage `uri`; identity-only\nexistence discovery uses [`GraphDiscoveryEntry`] instead.", + "description": "Submit exactly once. An uncertain response permits only GET observation of\nthis original operation identity, never a replacement ID or storage fallback.", "required": [ - "graph_id", - "uri", - "state", - "read_available", - "write_available", - "action" + "operation_id", + "graph", + "kind", + "limits" ], "properties": { - "action": { - "$ref": "#/components/schemas/GraphAvailabilityAction" + "graph": { + "type": "string", + "description": "An applied graph ID. Optimization always targets its main branch." }, - "failure": { - "oneOf": [ - { - "type": "null" + "kind": { + "$ref": "#/components/schemas/MaintenanceKind" + }, + "limits": { + "$ref": "#/components/schemas/MaintenanceLimits" + }, + "operation_id": { + "type": "string", + "description": "Canonical ledger ULID, next sequence and fresh ULID nonce, joined by `:`.", + "example": "01ARZ3NDEKTSV4RRFFQ69G5FAV:1:01ARZ3NDEKTSV4RRFFQ69G5FAW" + } + }, + "additionalProperties": false + }, + "MaintenanceResult": { + "oneOf": [ + { + "type": "object", + "required": [ + "operation_id", + "graph", + "invocation", + "report", + "outcome" + ], + "properties": { + "graph": { + "type": "string" + }, + "invocation": { + "$ref": "#/components/schemas/MaintenanceInvocation" + }, + "operation_id": { + "type": "string" + }, + "outcome": { + "type": "string", + "enum": [ + "success" + ] + }, + "report": { + "$ref": "#/components/schemas/OptimizeReport" + } + } + }, + { + "type": "object", + "required": [ + "operation_id", + "graph", + "invocation", + "code", + "message", + "outcome" + ], + "properties": { + "code": { + "type": "string" + }, + "graph": { + "type": "string" + }, + "invocation": { + "$ref": "#/components/schemas/MaintenanceInvocation" + }, + "message": { + "type": "string" + }, + "operation_id": { + "type": "string" + }, + "outcome": { + "type": "string", + "enum": [ + "refused" + ] + } + } + }, + { + "type": "object", + "required": [ + "operation_id", + "graph", + "invocation", + "publication", + "outcome" + ], + "properties": { + "graph": { + "type": "string" + }, + "invocation": { + "$ref": "#/components/schemas/MaintenanceInvocation" + }, + "operation_id": { + "type": "string" + }, + "outcome": { + "type": "string", + "enum": [ + "interrupted" + ] }, - { - "$ref": "#/components/schemas/GraphStartupFailure" + "publication": { + "$ref": "#/components/schemas/MaintenancePublication" } - ] - }, - "graph_id": { - "type": "string" - }, - "read_available": { - "type": "boolean", - "description": "Runtime availability; actor policy still gates every operation." - }, - "state": { - "$ref": "#/components/schemas/GraphAvailability" - }, - "uri": { - "type": "string" - }, - "write_available": { - "type": "boolean" - } - } - }, - "GraphListOutput": { - "type": "object", - "description": "Response from `GET /graphs`. Lists every graph registered with the\nserver in alphabetical order by `graph_id` (sorted server-side so\nclients get deterministic output across requests).", - "required": [ - "graphs" - ], - "properties": { - "graphs": { - "type": "array", - "items": { - "$ref": "#/components/schemas/GraphInfo" } } - } - }, - "GraphStartupFailure": { - "type": "string", - "description": "Bounded startup classification, without storage paths or error text.", - "enum": [ - "invalid_configuration", - "invalid_policy", - "invalid_external_blob_policy", - "open_failed", - "invalid_stored_queries" ] }, - "HealthOutput": { + "MaintenanceSchemaContract": { "type": "object", "required": [ - "status", - "version", - "internal_schema_version" + "source_hash", + "schema_ir_hash", + "schema_identity_domain", + "schema_identity_version" ], "properties": { - "internal_schema_version": { + "schema_identity_domain": { + "type": "string" + }, + "schema_identity_version": { "type": "integer", "format": "int32", - "description": "The internal-schema (storage-format) version this binary writes, the\nhighest it serves. Normal open serves every version from the binary's\nlowest supported one up to this one; a graph below that range is\nrefused until it is upgraded offline or rebuilt, and a graph above it\nneeds a newer binary.", "minimum": 0 }, - "source_version": { - "type": [ - "string", - "null" - ] - }, - "status": { + "schema_ir_hash": { "type": "string" }, - "version": { + "source_hash": { "type": "string" } - } + }, + "additionalProperties": false }, - "IngestOutput": { + "MaintenanceStatus": { "type": "object", "required": [ - "uri", - "branch", - "branch_created", - "mode", - "nodes", - "edges", - "total_entities", - "embedding_generation" + "ledger_id", + "next_sequence", + "outstanding_id" ], "properties": { - "actor_id": { - "type": [ - "string", - "null" - ] + "ledger_id": { + "type": "string", + "description": "Native ledger identity used in the next operation ID." }, - "base_branch": { + "next_sequence": { + "type": "integer", + "format": "int64", + "minimum": 1 + }, + "outstanding_id": { "type": [ "string", "null" ], - "description": "Base branch a fork was requested from (the request's `from`), echoed\neven when the branch already existed. `null` when `from` was absent." - }, - "branch": { - "type": "string" - }, - "branch_created": { - "type": "boolean" - }, - "commit": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/components/schemas/CommitOutput" - } - ] - }, - "edges": { - "type": "array", - "items": { - "$ref": "#/components/schemas/GraphBatchDeclarationOutput" - }, - "description": "Logical edge declarations touched by this load, sorted by name." - }, - "embedding_generation": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/components/schemas/LoadEmbeddingGeneration", - "description": "`unsupported` when a loaded node type declares `@embed`, including\nwhen all vectors were supplied; `null` otherwise." - } - ] - }, - "mode": { - "$ref": "#/components/schemas/LoadMode" - }, - "nodes": { - "type": "array", - "items": { - "$ref": "#/components/schemas/GraphBatchDeclarationOutput" - }, - "description": "Logical node declarations touched by this load, sorted by name." - }, - "total_entities": { - "type": "integer", - "minimum": 0 - }, - "uri": { - "type": "string" + "description": "Outstanding operation for this graph, or null. Status reads create no journal." } } }, - "IngestRequest": { + "MaintenanceStatusOutput": { "type": "object", "required": [ - "data" + "status", + "limits" ], "properties": { - "branch": { - "type": [ - "string", - "null" - ], - "description": "Target branch. Defaults to `main`. Without `from`, the branch must\nalready exist — a missing branch is a 404, never an implicit fork." - }, - "data": { - "type": "string", - "description": "NDJSON payload: one record per line, each shaped\n`{\"type\": \"\", \"data\": {...}}`.", - "example": "{\"type\": \"Person\", \"data\": {\"name\": \"Alice\", \"age\": 30}}\n{\"type\": \"Person\", \"data\": {\"name\": \"Bob\", \"age\": 25}}" + "limits": { + "$ref": "#/components/schemas/MaintenanceLimits" }, - "from": { - "type": [ - "string", - "null" - ], - "description": "Parent branch used to create `branch` if it does not exist. Branch\ncreation is opt-in by presence of this field; omit it to require an\nexisting branch." - }, - "mode": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/components/schemas/LoadMode", - "description": "How existing entities are handled. Defaults to `merge`." - } - ] + "status": { + "$ref": "#/components/schemas/MaintenanceStatus" } } }, - "InvokeStoredQueryOutput": { - "oneOf": [ - { - "$ref": "#/components/schemas/ReadOutput" - }, - { - "$ref": "#/components/schemas/ChangeOutput" - } - ], - "description": "Response for `POST /queries/{name}`: the read envelope for a stored\nread, or the mutation envelope for a stored mutation. Serialized\n**untagged**, so the wire shape is exactly [`ReadOutput`] or\n[`ChangeOutput`] — classification follows the stored query, not a\nwrapper field." - }, - "InvokeStoredQueryRequest": { + "MergeBaseOutput": { "type": "object", - "description": "Body for `POST /queries/{name}` — invokes the server-side stored query\nnamed in the path. The query source and name come from the registry,\nnever the body; only the runtime inputs are supplied here.", + "description": "The commit a merge of `source` into `target` would classify against, and\nthe two heads it was computed from.", + "required": [ + "source", + "target", + "source_commit_id", + "target_commit_id", + "merge_base" + ], "properties": { - "branch": { - "type": [ - "string", - "null" - ], - "description": "Branch to run against. Defaults to `main`; for a stored mutation the\nwrite targets this branch." + "merge_base": { + "$ref": "#/components/schemas/CommitOutput" }, - "expect_mutation": { - "type": [ - "boolean", - "null" - ], - "description": "The kind the caller expects: `Some(false)` for\n`omnigraph query `, `Some(true)` for `omnigraph mutate `.\nWhen set and it disagrees with the stored query's actual kind, the\nserver rejects the call (400) so the verb asserts the kind. `None`\n(the default) skips the check — preserving older clients and aliases." + "source": { + "type": "string" }, - "params": { - "description": "JSON object whose keys match the stored query's declared parameters." + "source_commit_id": { + "type": "string" }, - "snapshot": { - "type": [ - "string", - "null" - ], - "description": "Snapshot id to read from (read queries only — rejected for a stored\nmutation). Mutually exclusive with `branch`." + "target": { + "type": "string" + }, + "target_commit_id": { + "type": "string" } } }, - "KeyConflictOutput": { + "MergeConflictKind": { + "type": "string", + "enum": [ + "divergent_insert", + "divergent_update", + "delete_vs_update", + "orphan_edge", + "unique_violation", + "cardinality_violation", + "value_constraint_violation" + ] + }, + "MergeConflictOutput": { "type": "object", - "description": "A strict insert rejected because `entity_id` already names an entity in the\nselected node or edge type. The refused write changed nothing; an earlier\nstep of the same request, such as the branch a load forks, may already have\npublished, which the error's `disposition` states. Partial or ambiguous\nattempts surface `recovery_required` instead.", "required": [ "entity_kind", - "type_name" + "type_name", + "kind", + "message" ], "properties": { "entity_id": { @@ -14950,247 +16662,301 @@ "entity_kind": { "$ref": "#/components/schemas/EntityKind" }, + "kind": { + "$ref": "#/components/schemas/MergeConflictKind" + }, + "message": { + "type": "string" + }, "type_name": { "type": "string" } - } - }, - "LoadEmbeddingGeneration": { - "type": "string", - "description": "Load capability for a batch touching a node type with declared `@embed`.", - "enum": [ - "unsupported" - ] + } }, - "LoadExecutedOutput": { + "OptimizeInput": { "type": "object", - "description": "What a `load csv` statement's plan measured as it ran, beside the\nstatement's [`ChangeOutput`]: the source partitions and bytes it read, the\ncount its sink staged, and what each operator did. Absent for an append\nor merge load, which the keyed adapters run, and for every other change.", "required": [ - "partitions", - "rows", + "before_commit", + "report_max_bytes", + "schema_contract", + "source_tables", + "source_fragments", + "source_rows", "source_bytes", - "operators" + "source_files" ], "properties": { - "operators": { - "type": "array", - "items": { - "$ref": "#/components/schemas/LoadOperatorExecutedOutput" - }, - "description": "One entry per node of the executed plan in post-order, the source\nfirst." + "before_commit": { + "type": "string" }, - "partitions": { + "report_max_bytes": { "type": "integer", - "format": "int32", - "description": "Source partitions the plan read.", + "format": "int64", "minimum": 0 }, - "rows": { + "schema_contract": { + "$ref": "#/components/schemas/MaintenanceSchemaContract" + }, + "source_bytes": { "type": "integer", "format": "int64", - "description": "Entities the sink staged.", "minimum": 0 }, - "source_bytes": { + "source_files": { "type": "integer", "format": "int64", - "description": "Bytes of the captured source.", "minimum": 0 - } - } - }, - "LoadMode": { - "type": "string", - "description": "Shadow enum for documenting [`LoadMode`] in the OpenAPI schema.", - "enum": [ - "overwrite", - "append", - "merge" - ] - }, - "LoadOperatorExecutedOutput": { - "type": "object", - "description": "What one operator of an executed load plan did.", - "required": [ - "id", - "node", - "operator", - "rows", - "elapsed_compute_ns", - "wall_ns" - ], - "properties": { - "elapsed_compute_ns": { + }, + "source_fragments": { "type": "integer", "format": "int64", - "description": "Nanoseconds the operator was computing: the polls of its stream and of\nits workers, never the time it waited for its input or for storage.", "minimum": 0 }, - "id": { + "source_rows": { "type": "integer", "format": "int64", - "description": "The plan node's `id`, as the explain document numbers it.", "minimum": 0 }, - "node": { - "type": "string", - "description": "The plan node's kind: `CsvSource`, `Validate`, `FragmentWrite`." - }, - "operator": { - "type": "string", - "description": "The operator the node built." - }, - "partitions": { + "source_tables": { "type": "integer", "format": "int32", - "description": "The output partitions the operator ran, each its own stream: a\n`CsvSource`'s parsers, 1 for every other operator.", "minimum": 0 + } + }, + "additionalProperties": false + }, + "OptimizePendingIndex": { + "type": "object", + "required": [ + "type_key", + "property", + "reason" + ], + "properties": { + "property": { + "type": "string" }, - "rows": { - "type": "integer", - "format": "int64", - "description": "What the operator emitted; the sink counts what it staged.", - "minimum": 0 + "reason": { + "type": "string" }, - "slices": { - "type": "array", - "items": { - "$ref": "#/components/schemas/LoadSliceExecutedOutput" - }, - "description": "The slices of the source a `CsvSource` cut and parsed, in slice order;\nabsent for every other operator." + "type_key": { + "type": "string" + } + }, + "additionalProperties": false + }, + "OptimizePublication": { + "type": "object", + "required": [ + "before_manifest_version", + "intent_nonce", + "schema_contract", + "actor", + "created_at", + "report" + ], + "properties": { + "actor": { + "type": [ + "string", + "null" + ] }, - "wall_ns": { + "before_manifest_version": { "type": "integer", "format": "int64", - "description": "Nanoseconds from the operator's first poll to its end, waits included;\n`0` where no end was recorded.", "minimum": 0 + }, + "created_at": { + "type": "integer", + "format": "int64" + }, + "intent_nonce": { + "type": "string" + }, + "report": { + "$ref": "#/components/schemas/OptimizeReport" + }, + "schema_contract": { + "$ref": "#/components/schemas/MaintenanceSchemaContract" } - } + }, + "additionalProperties": false }, - "LoadSliceExecutedOutput": { + "OptimizeReport": { "type": "object", - "description": "What one slice of a `CsvSource` measured: a byte range of the capture\nparsed whole by one partition.", "required": [ - "slice", - "partition", - "rows", - "elapsed_compute_ns", - "start_ns", - "end_ns" + "before_commit", + "after_commit", + "tables", + "source_tables", + "source_fragments", + "source_rows", + "source_bytes", + "source_files" ], "properties": { - "elapsed_compute_ns": { - "type": "integer", - "format": "int64", - "description": "Nanoseconds the slice's parse and normalization took.", - "minimum": 0 + "after_commit": { + "type": "string" }, - "end_ns": { + "before_commit": { + "type": "string" + }, + "source_bytes": { "type": "integer", "format": "int64", - "description": "Nanoseconds after the source's first poll at which the slice's last\nbatch was sent.", "minimum": 0 }, - "partition": { + "source_files": { "type": "integer", - "format": "int32", - "description": "The partition that claimed the slice; which one varies between runs.", + "format": "int64", "minimum": 0 }, - "read_ns": { + "source_fragments": { "type": "integer", "format": "int64", - "description": "Nanoseconds the slice's partition waited for the source's bytes.", "minimum": 0 }, - "rows": { + "source_rows": { "type": "integer", "format": "int64", - "description": "The records the slice held.", "minimum": 0 }, - "slice": { + "source_tables": { "type": "integer", "format": "int32", - "description": "The slice's position in the source, from 0; a slice's rows are the\nsame on every run.", "minimum": 0 }, - "start_ns": { - "type": "integer", - "format": "int64", - "description": "Nanoseconds after the source's first poll at which the slice was\nclaimed.", - "minimum": 0 + "tables": { + "type": "array", + "items": { + "$ref": "#/components/schemas/OptimizeTableStats" + } } - } + }, + "additionalProperties": false }, - "MergeBaseOutput": { + "OptimizeSettlement": { "type": "object", - "description": "The commit a merge of `source` into `target` would classify against, and\nthe two heads it was computed from.", "required": [ - "source", - "target", - "source_commit_id", - "target_commit_id", - "merge_base" + "original_digest", + "lineage" ], "properties": { - "merge_base": { - "$ref": "#/components/schemas/CommitOutput" + "lineage": { + "$ref": "#/components/schemas/OptimizeSettlementLineage" }, - "source": { + "original_digest": { "type": "string" + } + }, + "additionalProperties": false + }, + "OptimizeSettlementLineage": { + "type": "object", + "required": [ + "graph_commit_id", + "branch", + "actor_id", + "merged_parent_commit_id", + "created_at" + ], + "properties": { + "actor_id": { + "type": [ + "string", + "null" + ] }, - "source_commit_id": { - "type": "string" + "branch": { + "type": [ + "string", + "null" + ] }, - "target": { - "type": "string" + "created_at": { + "type": "integer", + "format": "int64" }, - "target_commit_id": { + "graph_commit_id": { "type": "string" + }, + "merged_parent_commit_id": { + "type": [ + "string", + "null" + ] } - } + }, + "additionalProperties": false }, - "MergeConflictKind": { + "OptimizeSkipReason": { "type": "string", "enum": [ - "divergent_insert", - "divergent_update", - "delete_vs_update", - "orphan_edge", - "unique_violation", - "cardinality_violation", - "value_constraint_violation" + "drift_needs_repair" ] }, - "MergeConflictOutput": { + "OptimizeTableStats": { "type": "object", "required": [ - "entity_kind", - "type_name", - "kind", - "message" + "type_key", + "fragments_removed", + "fragments_added", + "committed", + "skipped", + "published_dataset_version", + "lance_head_version", + "pending_indexes" ], "properties": { - "entity_id": { + "committed": { + "type": "boolean" + }, + "fragments_added": { + "type": "integer", + "minimum": 0 + }, + "fragments_removed": { + "type": "integer", + "minimum": 0 + }, + "lance_head_version": { "type": [ - "string", + "integer", "null" - ] + ], + "format": "int64", + "minimum": 0 }, - "entity_kind": { - "$ref": "#/components/schemas/EntityKind" + "pending_indexes": { + "type": "array", + "items": { + "$ref": "#/components/schemas/OptimizePendingIndex" + } }, - "kind": { - "$ref": "#/components/schemas/MergeConflictKind" + "published_dataset_version": { + "type": [ + "integer", + "null" + ], + "format": "int64", + "minimum": 0 }, - "message": { - "type": "string" + "skipped": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/OptimizeSkipReason" + } + ] }, - "type_name": { + "type_key": { "type": "string" } - } + }, + "additionalProperties": false }, "ParamDescriptor": { "type": "object", diff --git a/skills/omnigraph/SKILL.md b/skills/omnigraph/SKILL.md index 154c426b1..479094d79 100644 --- a/skills/omnigraph/SKILL.md +++ b/skills/omnigraph/SKILL.md @@ -193,7 +193,7 @@ destination line. Legacy `--server`, `--store`, `--cluster`, | Data | `query`, `mutate`, `load --mode overwrite\|append\|merge`, `branch`, `commit`, `snapshot`, `export`, `blob`, `diff`, `changes`; direct or served targets | | Managed data | `graphs list` and common served graph commands, subject to endpoint capabilities and policy; no direct maintenance fallback | | Direct schema | `schema plan`; `schema apply` and `schema upgrade-system-columns` require standalone graph targets | -| Direct maintenance | `optimize`, `repair`, `cleanup`, `rebuild-full-text-indexes`; `upgrade --target ADDRESS` is standalone and offline | +| Maintenance | `optimize` on direct or enabled server targets; `repair`, `cleanup`, `rebuild-full-text-indexes` are direct; `upgrade --target ADDRESS` is standalone and offline | | Source-only | `lint --schema FILE --query FILE`, `cluster validate --config DIR`; no backend target | | Root control | `queries list/validate`, `policy validate/test/explain`, `cluster observe/force-unlock/upgrade-ledger`; direct root targets; `queries list` also reads one served graph | | Deployment | `cluster plan/apply --config DIR` on a direct root or server; `cluster status` uses only the target | diff --git a/skills/omnigraph/references/commands.md b/skills/omnigraph/references/commands.md index b9535d936..93655672a 100644 --- a/skills/omnigraph/references/commands.md +++ b/skills/omnigraph/references/commands.md @@ -293,8 +293,24 @@ Aliases are read-only. Invoke a served stored mutation with `omnigraph mutate ```bash omnigraph optimize --target $REPO --json +omnigraph optimize --target $SERVER --graph $GRAPH --no-wait --json +omnigraph optimize --target $SERVER --graph $GRAPH --operation-id '' --json ``` +Served optimize requires enabled bounded maintenance and applied `optimize` +permission on `main`; its actor comes from the credential. The CLI submits +once, prints its operation ID before sending, and then only observes that ID. +`--timeout` is 1–3600 seconds (default 300); expiry exits 5 without cancelling +work. `--operation-id` never submits, including when no receipt is recorded. +Retain the ID after uncertain delivery; never fall back to direct storage. +Managed targets do not yet support optimize. +Administrator recovery uses `optimize --target --graph +--operation-id --writers-stopped` only after predecessor and accepted-I/O +exclusion. It cannot override a held lock. It reconciles the original receipt +as interrupted unpublished/published; it never reruns compaction. An explicit +actor with current `optimize` permission is required. Unknown evidence stays +unresolved, with the original operation and lock identities retained. + Compacts fragments and reconciles declared scalar/vector index coverage without deleting retained versions. Lance 13 supports compaction of Blob-bearing data; null, From 69321d23099b608a984b893b803db5fa5fc241fd Mon Sep 17 00:00:00 2001 From: aaltshuler Date: Tue, 27 Oct 2026 12:58:49 +0200 Subject: [PATCH 2/4] docs(rfc): simplify served optimization ownership and outcomes --- docs/rfcs/2026-10-25-served-optimization.md | 510 ++++++++++++-------- 1 file changed, 298 insertions(+), 212 deletions(-) diff --git a/docs/rfcs/2026-10-25-served-optimization.md b/docs/rfcs/2026-10-25-served-optimization.md index 2f6796c57..ef559876b 100644 --- a/docs/rfcs/2026-10-25-served-optimization.md +++ b/docs/rfcs/2026-10-25-served-optimization.md @@ -7,223 +7,309 @@ implementation: in-progress authors: - aaltshuler created: 2026-10-25 -updated: 2026-10-26 -discussion: null +updated: 2026-10-27 +discussion: https://github.com/ModernRelay/omnigraph/pull/1177 supersedes: [] superseded_by: [] blocked_on: - - "Native qualification of resource bounds, retained data and shutdown settlement before deployment." + - "Align the implementation with the synchronous execution contract and qualify disconnect, publication and shutdown outcomes." + - "Qualify finite native work bounds and the supported deployment resource envelope before enabling execution." + - "Before managed enablement: amend LCY-04.2 and DEC-08-76 and qualify native enforcement of control-plane capacity allocation." --- # RFC: Served optimization -## Problem and scope - -`optimize` currently opens graph storage directly. A running cluster already -owns that storage, so an independent process cannot safely run this command -beside it. Expose the existing optimization through that owning server, retain -an exact operation identity, and let a disconnected client inspect its result. - -The first implementation operates on one graph's `main`. It can block writes -while holding existing native gates. Reads keep their admitted snapshots. -It does not introduce automatic scheduling, independent task workers, cleanup, -repair, history deletion or a second graph publication path. - -## Authority and execution - -Add the explicit graph policy action `optimize`, scoped to target branch -`main`. `admin` stays reserved; configuration management and graph membership -do not imply optimization permission. HTTP authenticates the actor. Native -cluster acceptance checks current applied policy; the engine entry point -checks the same action before work. No bearer bytes enter durable records. - -The server uses its existing cluster admission and owned-operation runtime. -One process gate serializes maintenance with configuration deployment. Native -acceptance checks the exact cluster owner and refuses outstanding deployment -work. Every accepted optimization remains owned after its HTTP waiter exits. -Shutdown uses the existing owned-operation and storage-settlement rules. - -The engine retains the existing schema, branch and sorted-table gates, exact +## Summary + +Expose `optimize` through the server that already owns the cluster. The server +owns safe execution and returns the exact result it observes. The control plane +owns scheduling, delivery, operational history and collected logs. The engine +keeps its existing snapshot, staging and graph-publication contract. + +The first slice is one synchronous request for one graph's `main`. It adds no +durable maintenance journal, scheduler, restart lookup or recovery command. +A lost response leaves the caller's outcome unknown; it never authorizes an +automatic retry. This is a deliberate limit on the execution contract. + +The implementation in [#1177](https://github.com/ModernRelay/omnigraph/pull/1177) +currently includes a stronger durable invocation protocol. It must be reduced +to this revised design before it can claim conformance. This document changes +the proposal; it does not claim the implementation already matches it. + +## Motivation + +The existing direct `optimize` opens graph storage. A running cluster already +owns that storage, so an independent process cannot safely optimize beside it. +Operators need to run the existing operation through that owner. + +Durable job tracking is a separate requirement. Making receipt retention a +prerequisite for serving creates an availability dependency unrelated to whether +the graph is safe. A manual operation does not need its own sequence ledger, +publication-witness protocol or offline recovery workflow to be useful. + +## User and operational behavior + +For a configured server target, the proposed command is: + +```bash +omnigraph optimize --target prod --graph knowledge +omnigraph optimize --target prod --graph knowledge --json +``` + +The command waits for this attempt's result. It reports the accepted input +commit, its resulting commit or unchanged commit, table statistics, skips, +deferred indexes and any optional-work failures. It may block writes while +holding existing gates; admitted reads retain their snapshots. + +The existing direct command remains available under normal exclusive native +ownership. A remote refusal never falls back to direct storage. Existing +target selection, actor assertion and result-output conventions apply. There +are no new `--no-wait`, `--operation-id` or `--writers-stopped` modes. + +| Observation | Meaning and caller action | +|---|---| +| Definitive refusal before effects | This request did not optimize. Correct the reported cause before making another request. | +| Completed result | Use this attempt's exact report, including no-work and optional-work outcomes. | +| Lost response or observation timeout | Execution may still be running or may have completed. Record unknown; do not automatically resubmit. | +| Native failure | Preserve its typed effect and settlement evidence; an error alone does not prove that nothing published. | + +A later explicitly requested optimization is a new attempt against a fresh +accepted snapshot, after native admission permits it. It may perform additional +work or publish another commit. It is not replay, recovery or proof that the +previous attempt failed. A changed graph head or equal logical data does not +identify the old attempt's result. + +## Design + +### Responsibility boundary + +| Owner | Responsibility | +|---|---| +| Control plane | Scheduling, fleet capacity allocation, delivery policy, durable workflow history, log collection/retention and operator presentation. | +| Server | Authentication, local admission, ownership of execution through disconnect, bounded execution status and exact responses. | +| Engine | Authorization at the native entry point, accepted snapshot, physical optimization, graph publication and typed storage outcomes. | + +The control plane records intent, delivery attempts and responses it actually +observed. Its records are authoritative for its workflow, not native acceptance +or publication. An unobserved result remains unknown. Self-hosted callers can +use the same native endpoint without deploying a control plane. + +The server emits ordinary correlated diagnostic events and metrics through the +existing observability facilities. It does not persist a second operational +history or manage log delivery. Loss of an observational record is neither +proof of safety nor a reason to close otherwise valid native admission. +This minimizes management overhead; physical optimization still runs in the +engine. Moving that computation to independent workers is out of scope. + +### Native execution and outcomes + +Add the graph policy action `optimize`, scoped to `main`. Installed policies +must explicitly grant it; `admin`, configuration management and membership do +not imply it. HTTP resolves the bearer to an actor; cluster admission checks +current applied policy and the engine checks the same action before effects. +Keep normal access discovery and the existing `--as` credential-actor assertion. +An assertion never supplies a trusted actor. Do not retain bearer bytes. + +Reuse the existing cluster admission, owned-operation runtime and deployment +gate. Admit at most one optimization per cluster and serialize it with +configuration deployment. Refuse another optimization while that slot or the +deployment gate is busy; ordinary writes keep their existing gate behavior. +Accepted work remains owned after its HTTP waiter exits. Disconnect and +observation deadlines end waiting, not native execution. + +The engine retains its schema, branch and sorted-table gates, one coherent accepted snapshot, detached staging and single graph-manifest publication. -Its report captures the before/after graph commit while those gates remain -held, plus exact table statistics, skips and deferred indexes. A report must -not substitute a later head read for this operation's publication. - -## Durable invocation and lookup - -Store a separate bounded native invocation journal at -`__cluster/maintenance.json`. This is an operation record, not graph authority -or another ownership lock. Storage uses the admitted root's existing adapter, -conditional writes and I/O scope. It never modifies deployment outcomes. - -An operation ID is `::`; the ledger ID and -nonce are canonical ULIDs and the sequence is a canonical positive integer. -Maintenance has its own increasing sequence. Read-only status exposes the -next sequence but reserves nothing. Acceptance consumes one sequence and -persists the exact graph, kind, options, budget and authenticated actor before -any effect. It returns a single-use in-process execution token. No caller can -construct an accepted token from the serialized journal. - -After the engine completes source preflight under its graph gates, it -conditionally binds the exact input commit and source inventory to that -invocation before any physical rewrite. Success must match this binding. -A failed binding write retains uncertainty and never permits execution. -The invocation also records its applied configuration revision and schema -contract; these identify admission, while the immutable input commit binds -the actual graph schema and table pins used by optimization. - -Only one outstanding invocation is allowed per cluster. A duplicate identity -and matching input/actor returns the recorded status without execution. -Different input or actor refuses. A different nonce at a retained sequence -refuses. A consumed sequence whose result was evicted reports expired, -unknown acceptance/outcome; it never becomes executable again. Future -sequences and another ledger are refused. - -Completion conditionally records the exact native report or a proven -pre-effect refusal. A refusal consumes its ID and releases the slot; an -uncertain failure keeps it outstanding. Keep at most 32 -terminal reports, at most 1 MiB each and 4 MiB combined. The complete journal -is limited to 8 MiB. Eviction only removes terminal result payloads; the -monotonic sequence retains the no-replay boundary. Outstanding work cannot -be evicted. A result larger than its retention bound leaves uncertainty rather -than publishing a truncated success. - -Before physical effects, the engine derives a conservative report bound from -the exact accepted table and index strings, all supported pending-index -outcomes and maximal numeric widths. Cluster admission includes the complete -invocation/result wrapper, prepared witness and interrupted-result shapes in -the reservation. An envelope that cannot fit refuses before physical work. -The final exact-size check remains a defensive guard. No report is truncated, -and no input/output size ratio substitutes for this proof. - -A crash or storage error between invocation acceptance, graph publication and -result recording leaves an outstanding operation. Lookup labels it unresolved -when no local owner exists. Neither a new process nor another request replays -it or infers success from equal data or a changed head. It preserves its evidence -and refuses new maintenance, deployment and writable process admission. Even -after explicit predecessor exclusion, a successor cannot start ordinary -serving or direct graph operations while the journal is unresolved. The -original process can expose its exact receipt while its admission remains -available. Explicit recovery takes a restricted admission for the original -operation after the administrator establishes predecessor and accepted-I/O -exclusion and handles the exact old lock. It cannot serve requests, deploy -configuration or rerun optimization. - -Before publication, retain the exact native publication witness and prepared -report. Recovery compares that witness against native graph authority. No -prepared witness means graph publication was never attempted. A prepared -candidate's absence alone does not exclude a delayed storage write: use the -existing native no-effect settlement publication to resolve that race, then -validate the exact winner. Missing, conflicting or unclassifiable evidence -leaves the original invocation unresolved. A recovered receipt explicitly -reports interrupted unpublished or interrupted published work; publication -does not claim subsequent derived-artifact work completed. Detached output -remains subject to ordinary native cleanup safety. Recovery never deletes it. - -Current `optimize` permission authorizes status for the requested graph. The -exact authenticated initiator may retrieve its retained individual receipt -after permission revocation, without gaining graph data or other operations. -Missing or expired ownership evidence cannot grant this exception. - -## Bounds and enablement - -Served optimization is explicitly enabled at boot; it is off by default. -The optional `OMNIGRAPH_MAINTENANCE_CONFIG` names a local YAML file (at most -4 KiB) containing positive `max_tables`, `max_fragments`, `max_rows`, -`max_source_bytes` and `max_files`. Embedders may provide the same limits -through the server builder. There are no implicit defaults that enable work. -The server chooses finite ceilings for source tables, fragments, rows, bytes, -file references and dataset concurrency. Requests may lower those ceilings, never raise them. -The first implementation uses one dataset at a time. Limits are validated -before durable acceptance and the complete accepted work is checked before -the first physical effect, including dropped-column rewrite tasks. - -Source accounting includes `__manifest` and all pinned table manifests, data and deletion -files, index-file inventories and live managed Blob lengths. It conservatively -counts whole accepted tables, including dropped-column rewrites; Blob lengths -may count bytes already inside a data file, and each nonempty managed Blob -counts as a file reference even when values share a packed file. External Blob -payloads are not read by compaction. Missing file sizes use exact object HEADs; -unknown index inventories or physical formats, inherited storage bases, system -indexes, overlays and unsupported Blob layouts refuse before effects. Index -accounting inspects native metadata before reader compatibility filtering; -an unsupported index cannot disappear from the accepted inventory. -No historical prefix is listed. -Unknown sizes or unsupported accounting refuse; a per-dataset concurrency -setting alone is not a whole-operation limit. A bounded input is not a claim -of a hard process memory, CPU or storage-rate cap. Live enablement additionally -requires process/platform resource isolation and measured limits. An elapsed -HTTP deadline never interrupts an uncertain storage write. - -This accounting was checked against the pinned Lance 13.0.0 source: -[`CompactionOptions`, planning and task execution](https://docs.rs/crate/lance/13.0.0/source/src/dataset/optimize.rs), -[`Fragment` and `DataFile` metadata](https://docs.rs/crate/lance-table/13.0.0/source/src/format/fragment.rs), -[`IndexMetadata.files`](https://docs.rs/crate/lance-table/13.0.0/source/src/format/index.rs), -the native [`deletion_file_path`](https://docs.rs/crate/lance-table/13.0.0/source/src/io/deletion.rs), -[`Dataset::object_store` and `branch_location`](https://docs.rs/crate/lance/13.0.0/source/src/dataset.rs), -and [`ObjectStore::size`](https://docs.rs/crate/lance-io/13.0.0/source/src/object_store.rs). -Lance's own compaction byte option omits Blob-v2 payloads and can select only -part of the work. It therefore does not substitute for the complete accepted -inventory checked here. These source checks establish the accounting contract, -not a measured process-memory or throughput limit. - -Index reconciliation defers an existing vector index only when it has no -non-null vectors. With positive input, native Lance chooses its supported -training layout. Statistics, storage and index builder errors propagate; they -cannot masquerade as insufficient training data. -New declared vector indexes with no non-null rows retain their existing pending -status. Full-text reconciliation keeps its explicit rebuild boundary. - -No externally scheduled capacity allocation is defined here. A deployment -requiring a fleet-wide budget must keep this capability disabled until native -admission validates that allocation on every entry point. A proxy-only check -is insufficient. Root-bound signed-token and OIDC identity profiles refuse -new optimization at boot and native request admission while this allocation -contract is absent. Disabling execution retains original-ID lookup. -Independent workers additionally require retained immutable -inputs, serialized bounded plans, output-only credentials and native result -validation/publication; none is implied by this endpoint. - -## HTTP and CLI - -Use the shared operation registry and existing HTTP version: - -- `GET /cluster/maintenance?graph=`: authorized read-only status and next ID - components for one graph. -- `POST /cluster/maintenance`: exact operation ID, graph, `optimize` kind and - bounded options; return after durable acceptance or a recorded refusal. -- `GET /cluster/maintenance/{id}?graph=`: exact original result/status - lookup with the original graph, without scanning other graph policies. - -Malformed input, denied policy, disabled capability and exhausted admission -refuse before physical effects. An ambiguous POST response is not a refusal. -Clients retain the original ID and use GET; no automatic POST replay. - -These cluster maintenance routes have no MCP binding in this slice. Their -durable invocation and recovery contract is separate from the non-streaming -graph operations exposed as MCP tools. The shared registry records this -explicit transport boundary. - -Extend the existing `optimize` command and target resolution. Remote execution -waits by default; `--no-wait` returns the known operation ID. -`--operation-id ` observes that original operation without submitting it. -`--timeout` bounds observation only. Direct storage retains its existing -command and output; remote refusal never falls back to direct storage. -For administrator recovery, `optimize --operation-id --writers-stopped` -requires an explicit direct-root target and graph. The attestation means the -prior process and accepted graph/control I/O are excluded, as for deployment -recovery; it never overrides a held lock. This operation reconciles the -original receipt and never executes a new optimization. - -## Evidence required - -Extend existing maintenance, policy, cluster, server and CLI test owners. -Cover denied and allowed actors, complete bounds before effects, concurrent -reads and write contention, identical/mismatched duplicates, response loss, -result eviction, accepted-operation interruption, shutdown and restart lookup. -Compare logical rows, schema, Blob values, index behavior, retained history and -live branches before and after. Test failures before and after publication -without treating a lost response as permission to execute again. - -Local tests establish the candidate's contract. They do not qualify cloud -storage settlement, fleet admission or production resource ceilings. +Capture the exact before/after commits as part of that attempt, while the +relevant gates are held; never infer the result from a later head read. +Direct and served entry points share these semantics. Resource limits constrain +work; their presence must not select a different publication or error protocol. + +Separate known graph publication from optional derived-artifact work. A settled +artifact failure is reported explicitly without relabeling a known commit as +unknown. Genuine uncertain I/O, including a possibly accepted write to a mutable +cache object, still needs the existing settlement and containment rules. Never +discard that uncertainty merely because the data is derived. +Key-scoped storage uncertainty does not itself close process admission; +containment remains limited to the cases defined by the existing storage-outcome +contract, rather than every failure after acceptance. + +Shutdown and successor admission follow the existing +[owned-operation](2026-09-30-owned-server-operations.md) and +[storage-outcome](2026-10-19-storage-outcomes-and-restartable-ownership.md) +contracts. A dropped future does not exclude accepted I/O or a predecessor. +There is no additional startup blocker for a missing maintenance receipt, no +neutral graph commit solely to settle one, and no maintenance-specific unlock. + +### HTTP, receipts and status + +Add `POST /graphs/{graph_id}/optimize` through the shared operation registry, +typed API and [HTTP conventions](2026-10-19-http-api-conventions.md). The graph +is addressed once, in the path. The initial operation uses the existing optimize +behavior without new request options; clients do not copy server ceilings into +a maintenance envelope. +Return the exact completed report after durable publication, or the typed +failure. A no-work result names the unchanged commit. Do not return `202` as a +promise of durable acceptance. + +Here, a receipt means the completed response observed by the caller. It is not +a server-persisted record. The first slice exposes completion through that +response and keeps execution status in the existing owned-operation runtime; +it adds no per-invocation lookup endpoint. A request correlation ID supplies +identification only, not idempotency, deduplication or restart recovery. +Clients and control-plane delivery code must disable automatic POST retries +after ambiguous dispatch. Native status may be lost at restart. + +MCP exposure and asynchronous lookup are deferred. If reconnect lookup becomes +necessary, a separate bounded, process-local extension can retain active and +terminal statuses scoped to the server incarnation. Missing, evicted or old +statuses must mean unknown, never not-executed, and lookup must never execute +work. That extension must define its limits and authorization before shipping; +this RFC does not require a generic jobs API to expose optimize. + +### Bounds and enablement + +Execution is disabled until explicitly configured and qualified. Keep typed +limits under the server's existing configuration/builder ownership, without a +maintenance-only YAML file or parser. The server selects finite ceilings for +tables, fragments, physical rows, source bytes and file references; callers +cannot raise them. Start with one dataset and one compaction task in flight. +Bound metadata enumeration and result construction as well as physical work. + +Validate the complete accepted source inventory before the first rewrite, +including dropped-column materialization, catalog work, table manifests, +index files, deletion files and managed Blob payloads. External Blob references +remain references; their payloads are not fetched. Count before exceeding the +limit, rather than materializing an unbounded inventory and checking afterwards. +Unknown sizes or unsupported layouts refuse. Preserve the current full-text +rebuild boundary; index/statistics/storage errors must not masquerade as +insufficient vector training data. Do not add historical-prefix scans or output +truncation. + +Finite source ceilings and concurrency do not establish hard memory, CPU, +scratch-space or I/O-rate guarantees. Native work and completion resources need +measured limits and process/platform isolation for the supported envelope, +including competition with foreground requests. Existing request-count and +request-byte admission is not a substitute. Ordinary bounded result delivery +remains necessary; the durable-journal report reservation and recovery-envelope +machinery are not part of this contract. + +Managed execution additionally requires native enforcement of control-plane +capacity allocation on every entry point, including direct remote CLI calls. +A proxy check alone is insufficient. State support through explicit deployment +capability and resource admission, not an inference from static-token, signed +token or OIDC authentication. Until qualified, managed execution stays disabled. + +## Invariants + +The [architectural invariants](../dev/invariants.md) remain unchanged: one +accepted snapshot, detached pre-publication effects, one graph publication, +engine authorization, bounded work and typed uncertain outcomes. Derived indexes +and caches affect cost, never logical correctness. Existing ownership and I/O +settlement remain necessary even when observational status is missing. + +There is no second storage writer, manifest-derived job queue, custom WAL, +shadow publication authority, implicit retry, or inline content-write rebuild. +The control plane cannot bypass native policy, resource admission or fencing. +Cleanup, repair, history deletion and isolated workers remain outside this RFC. + +## Compatibility and reversibility + +This design adds no graph format or durable maintenance object. Adding the +`optimize` action requires policy and access-discovery updates for direct and +served callers. Removing the endpoint later leaves ordinary graph commits and +detached output governed by existing retention and cleanup rules. + +The candidate in #1177 has not established a supported wire or disk contract. +Remove its maintenance journal, sequence protocol, lookup/recovery routes, +receipt-only admission blocks and recovery CLI flags before shipping. Do not +silently delete an outstanding journal in an experimental deployment: exclude +its owner and settle its native work using the candidate's recovery procedure +before switching binaries. Migration of such experimental roots is not supplied +by this documentation change. + +The control-plane specifications +[LCY-04.2](https://github.com/ModernRelay/og-control-plane/blob/58a29d6dd4ea00cef84829f03181ffe740d01974/specs/08-LCY-Lifecycle.md#L702) +and [DEC-08-76](https://github.com/ModernRelay/og-control-plane/blob/58a29d6dd4ea00cef84829f03181ffe740d01974/specs/02-DEC-Decisions.md#L3601) +currently require durable native invocation identity, acceptance, results and +original-ID reconciliation. Amend those requirements before claiming managed +compatibility with this weaker observation contract. Moving the same guarantees +into control-plane bookkeeping would not satisfy them. + +## Alternatives + +- **Keep direct-only optimization:** requires stopping the owner for routine + physical maintenance; does not solve the operational problem. +- **Durable native invocation journal:** supports original-ID lookup and + publication reconciliation across crashes, with unresolved or partial outcomes + when evidence is insufficient. It adds persistent identities, retention, + recovery and an availability dependency on receipt settlement. Defer until a + concrete use case requires that stronger guarantee. +- **Reuse the deployment ledger:** couples configuration convergence to physical + optimization without removing durable invocation complexity. +- **Control-plane-only execution receipts:** cannot prove native publication + when a response was lost. Keep workflow history there and preserve unknowns. +- **Process-local asynchronous status:** useful for reconnecting observers, but + adds retention, authorization and incarnation semantics. Defer until the + synchronous slice demonstrates that need. + +## Evidence and tests + +Extend the existing engine `integration/maintenance.rs`, policy, cluster +admission, server operation/runtime and CLI maintenance owners. Prove: + +- Denied actors, unsupported deployments, exhausted bounds and competing work + refuse before physical effects; direct calls cannot bypass engine policy. +- Disconnect and timeout leave execution owned, with no client resubmission. + Shutdown and predecessor exclusion preserve native I/O settlement. +- Success reports this attempt's exact commit; no-work reports the unchanged + commit. Lost responses and crashes never become inferred success or failure. +- Known publication plus settled optional failure preserves both facts; + uncertain publication or optional I/O retains the required native fencing. +- Rows, schema, Blob values, index behavior, retained history and live branches + are preserved. Use GQT for expressible logical assertions; retain Rust owners + for physical mechanisms, concurrency and faults. + +Update the registry, OpenAPI, CLI contracts and DST storage-cost golden with +implementation changes. Verify that no receipt lookup, journal write or +maintenance-specific startup gate remains. Qualify resource and live-cloud +settlement separately; local correctness tests cannot establish those limits. +Measure large Blob rows, wide/high-dimensional rows, many small fragments and +index rebuilds under the intended allocation. Publish the tested limits. + +The candidate's source accounting was inspected against unmodified Lance +13.0.0: [compaction planning/execution](https://docs.rs/crate/lance/13.0.0/source/src/dataset/optimize.rs), +[fragment metadata](https://docs.rs/crate/lance-table/13.0.0/source/src/format/fragment.rs), +[index inventories](https://docs.rs/crate/lance-table/13.0.0/source/src/format/index.rs) +and [object-store sizing](https://docs.rs/crate/lance-io/13.0.0/source/src/object_store.rs). +Lance's compaction byte option alone excludes Blob-v2 payload accounting and +does not bound all optimization work. Reuse qualified native mechanisms rather +than introducing another transaction or recovery protocol. Tests of the prior +journal design are not evidence that this revised contract is implemented. + +## Rollout + +1. Review this reduced contract and align #1177's code, tests, OpenAPI and user + documentation. Keep execution disabled during qualification. +2. Enable the synchronous self-hosted slice only for the measured supported + resource envelope. Preserve explicit unknown outcomes and existing shutdown + behavior; publish the policy change and limits with the implementation. +3. Amend the control-plane specifications and qualify native capacity admission + before enabling managed calls. Keep scheduling and operational history in + the control plane. This phase does not gate the self-hosted slice. + +## Unresolved questions + +No additional design decision is required for the synchronous scope. The +implementation, resource and managed-integration evidence gates are listed in +`blocked_on`. Durable receipts, asynchronous lookup and worker offload require +separate justification and are not implied follow-on commitments. + +## Decision log + +- **2026-10-27:** Narrow the initial candidate to safe execution through the + existing owner. Replace durable invocation/lookup and offline recovery with + synchronous exact results and explicit unknown outcomes. Assign orchestration, + history and log collection to the control plane; retain native authorization, + resource admission, publication and settlement. Record the control-plane + specification changes needed before managed enablement. The implementation + still needs to be aligned; this revision does not accept its earlier protocol. From 00559b8b871c0cab53f3d4650060d5998619f291 Mon Sep 17 00:00:00 2001 From: aaltshuler Date: Tue, 27 Oct 2026 13:03:11 +0200 Subject: [PATCH 3/4] docs(rfc): explain served optimization in plain language --- docs/rfcs/2026-10-25-served-optimization.md | 43 +++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/docs/rfcs/2026-10-25-served-optimization.md b/docs/rfcs/2026-10-25-served-optimization.md index ef559876b..a25cf5a04 100644 --- a/docs/rfcs/2026-10-25-served-optimization.md +++ b/docs/rfcs/2026-10-25-served-optimization.md @@ -19,6 +19,49 @@ blocked_on: # RFC: Served optimization +## In plain language + +**Purpose:** let an operator optimize a graph while its server stays running. + +**The problem:** as data changes, its storage can become fragmented. +`optimize` reorganizes data files and maintains supported indexes so the engine +can read them more efficiently, while preserving the graph's data. Today the +command opens storage itself. Running it beside a server would introduce a +second writer that the server does not control. + +**The proposed solution:** send the request to the server that already manages +the graph. That server checks permission and available capacity, runs the +existing engine operation, and returns a report of what happened. The command +waits for that report. Reads can continue; writes may wait while optimization +holds the resources it needs. + +**Key design decisions:** + +- **Keep management in the control plane.** For managed clusters, it decides + when to request work and keeps scheduling records, operational history and + collected logs. The server checks each request and owns its execution; the + engine performs the storage changes safely. A self-hosted operator can call + the server directly without a control plane. +- **Keep the server interface small.** One request optimizes one graph's + `main` branch and returns its result. The server does not gain a persistent + jobs database, automatic scheduler or maintenance recovery command. Looking + up an individual operation after disconnect is outside this first slice. +- **Reuse the existing safety rules.** The engine makes the optimized graph + visible as one complete change. The server enforces permissions and work + limits, and admits only one optimization at a time per cluster. Optimization + still consumes server resources; this proposal does not move the computation + to the control plane or promise maintenance without impact on other work. +- **Be honest when the answer is lost.** Losing the connection does not cancel + the work. If the caller never receives the result, the outcome is unknown; + neither the CLI nor the control plane automatically submits it again. The + server does not promise to recover that request's report after a restart. + +**The tradeoff:** this gives operators a safe way to request optimization with +little new server machinery. It deliberately gives up guaranteed recovery of +every request's result after a crash. The control plane can keep a history of +what it requested and observed, but cannot turn a missing answer into proof +that the work succeeded or failed. + ## Summary Expose `optimize` through the server that already owns the cluster. The server From a49658fb8c9a0251f7d3252ec49948bc98ebf247 Mon Sep 17 00:00:00 2001 From: aaltshuler Date: Tue, 27 Oct 2026 13:05:45 +0200 Subject: [PATCH 4/4] docs(rfc): move served optimization design to separate PR --- docs/dev/control-plane.md | 8 +- docs/rfcs/2026-10-25-served-optimization.md | 358 -------------------- docs/rfcs/README.md | 1 - 3 files changed, 5 insertions(+), 362 deletions(-) delete mode 100644 docs/rfcs/2026-10-25-served-optimization.md diff --git a/docs/dev/control-plane.md b/docs/dev/control-plane.md index 08a0e37f0..2fb668d93 100644 --- a/docs/dev/control-plane.md +++ b/docs/dev/control-plane.md @@ -96,10 +96,12 @@ restricted original-ID admission after writer/I/O exclusion and classifies the exact native publication; its journal is never graph authority. An absent candidate alone cannot settle a delayed write. Native conditional no-effect publication resolves that race without replaying optimization. -The [served optimization RFC](../rfcs/2026-10-25-served-optimization.md) defines -this candidate contract and its qualification limits; the +The revised [served optimization RFC](https://github.com/ModernRelay/omnigraph/pull/1186) +is reviewed separately and proposes a smaller execution contract. This +implementation candidate still needs to be aligned with that design. The [maintenance guide](../user/operations/maintenance.md#server-enablement) describes -enablement and observation. It provides no scheduler or independent worker. +the candidate's enablement and observation. It provides no scheduler or +independent worker. ## Deployment ledger diff --git a/docs/rfcs/2026-10-25-served-optimization.md b/docs/rfcs/2026-10-25-served-optimization.md deleted file mode 100644 index a25cf5a04..000000000 --- a/docs/rfcs/2026-10-25-served-optimization.md +++ /dev/null @@ -1,358 +0,0 @@ ---- -rfc: "2026-10-25-served-optimization" -title: "Served optimization" -track: maintainer -status: draft -implementation: in-progress -authors: - - aaltshuler -created: 2026-10-25 -updated: 2026-10-27 -discussion: https://github.com/ModernRelay/omnigraph/pull/1177 -supersedes: [] -superseded_by: [] -blocked_on: - - "Align the implementation with the synchronous execution contract and qualify disconnect, publication and shutdown outcomes." - - "Qualify finite native work bounds and the supported deployment resource envelope before enabling execution." - - "Before managed enablement: amend LCY-04.2 and DEC-08-76 and qualify native enforcement of control-plane capacity allocation." ---- - -# RFC: Served optimization - -## In plain language - -**Purpose:** let an operator optimize a graph while its server stays running. - -**The problem:** as data changes, its storage can become fragmented. -`optimize` reorganizes data files and maintains supported indexes so the engine -can read them more efficiently, while preserving the graph's data. Today the -command opens storage itself. Running it beside a server would introduce a -second writer that the server does not control. - -**The proposed solution:** send the request to the server that already manages -the graph. That server checks permission and available capacity, runs the -existing engine operation, and returns a report of what happened. The command -waits for that report. Reads can continue; writes may wait while optimization -holds the resources it needs. - -**Key design decisions:** - -- **Keep management in the control plane.** For managed clusters, it decides - when to request work and keeps scheduling records, operational history and - collected logs. The server checks each request and owns its execution; the - engine performs the storage changes safely. A self-hosted operator can call - the server directly without a control plane. -- **Keep the server interface small.** One request optimizes one graph's - `main` branch and returns its result. The server does not gain a persistent - jobs database, automatic scheduler or maintenance recovery command. Looking - up an individual operation after disconnect is outside this first slice. -- **Reuse the existing safety rules.** The engine makes the optimized graph - visible as one complete change. The server enforces permissions and work - limits, and admits only one optimization at a time per cluster. Optimization - still consumes server resources; this proposal does not move the computation - to the control plane or promise maintenance without impact on other work. -- **Be honest when the answer is lost.** Losing the connection does not cancel - the work. If the caller never receives the result, the outcome is unknown; - neither the CLI nor the control plane automatically submits it again. The - server does not promise to recover that request's report after a restart. - -**The tradeoff:** this gives operators a safe way to request optimization with -little new server machinery. It deliberately gives up guaranteed recovery of -every request's result after a crash. The control plane can keep a history of -what it requested and observed, but cannot turn a missing answer into proof -that the work succeeded or failed. - -## Summary - -Expose `optimize` through the server that already owns the cluster. The server -owns safe execution and returns the exact result it observes. The control plane -owns scheduling, delivery, operational history and collected logs. The engine -keeps its existing snapshot, staging and graph-publication contract. - -The first slice is one synchronous request for one graph's `main`. It adds no -durable maintenance journal, scheduler, restart lookup or recovery command. -A lost response leaves the caller's outcome unknown; it never authorizes an -automatic retry. This is a deliberate limit on the execution contract. - -The implementation in [#1177](https://github.com/ModernRelay/omnigraph/pull/1177) -currently includes a stronger durable invocation protocol. It must be reduced -to this revised design before it can claim conformance. This document changes -the proposal; it does not claim the implementation already matches it. - -## Motivation - -The existing direct `optimize` opens graph storage. A running cluster already -owns that storage, so an independent process cannot safely optimize beside it. -Operators need to run the existing operation through that owner. - -Durable job tracking is a separate requirement. Making receipt retention a -prerequisite for serving creates an availability dependency unrelated to whether -the graph is safe. A manual operation does not need its own sequence ledger, -publication-witness protocol or offline recovery workflow to be useful. - -## User and operational behavior - -For a configured server target, the proposed command is: - -```bash -omnigraph optimize --target prod --graph knowledge -omnigraph optimize --target prod --graph knowledge --json -``` - -The command waits for this attempt's result. It reports the accepted input -commit, its resulting commit or unchanged commit, table statistics, skips, -deferred indexes and any optional-work failures. It may block writes while -holding existing gates; admitted reads retain their snapshots. - -The existing direct command remains available under normal exclusive native -ownership. A remote refusal never falls back to direct storage. Existing -target selection, actor assertion and result-output conventions apply. There -are no new `--no-wait`, `--operation-id` or `--writers-stopped` modes. - -| Observation | Meaning and caller action | -|---|---| -| Definitive refusal before effects | This request did not optimize. Correct the reported cause before making another request. | -| Completed result | Use this attempt's exact report, including no-work and optional-work outcomes. | -| Lost response or observation timeout | Execution may still be running or may have completed. Record unknown; do not automatically resubmit. | -| Native failure | Preserve its typed effect and settlement evidence; an error alone does not prove that nothing published. | - -A later explicitly requested optimization is a new attempt against a fresh -accepted snapshot, after native admission permits it. It may perform additional -work or publish another commit. It is not replay, recovery or proof that the -previous attempt failed. A changed graph head or equal logical data does not -identify the old attempt's result. - -## Design - -### Responsibility boundary - -| Owner | Responsibility | -|---|---| -| Control plane | Scheduling, fleet capacity allocation, delivery policy, durable workflow history, log collection/retention and operator presentation. | -| Server | Authentication, local admission, ownership of execution through disconnect, bounded execution status and exact responses. | -| Engine | Authorization at the native entry point, accepted snapshot, physical optimization, graph publication and typed storage outcomes. | - -The control plane records intent, delivery attempts and responses it actually -observed. Its records are authoritative for its workflow, not native acceptance -or publication. An unobserved result remains unknown. Self-hosted callers can -use the same native endpoint without deploying a control plane. - -The server emits ordinary correlated diagnostic events and metrics through the -existing observability facilities. It does not persist a second operational -history or manage log delivery. Loss of an observational record is neither -proof of safety nor a reason to close otherwise valid native admission. -This minimizes management overhead; physical optimization still runs in the -engine. Moving that computation to independent workers is out of scope. - -### Native execution and outcomes - -Add the graph policy action `optimize`, scoped to `main`. Installed policies -must explicitly grant it; `admin`, configuration management and membership do -not imply it. HTTP resolves the bearer to an actor; cluster admission checks -current applied policy and the engine checks the same action before effects. -Keep normal access discovery and the existing `--as` credential-actor assertion. -An assertion never supplies a trusted actor. Do not retain bearer bytes. - -Reuse the existing cluster admission, owned-operation runtime and deployment -gate. Admit at most one optimization per cluster and serialize it with -configuration deployment. Refuse another optimization while that slot or the -deployment gate is busy; ordinary writes keep their existing gate behavior. -Accepted work remains owned after its HTTP waiter exits. Disconnect and -observation deadlines end waiting, not native execution. - -The engine retains its schema, branch and sorted-table gates, one coherent -accepted snapshot, detached staging and single graph-manifest publication. -Capture the exact before/after commits as part of that attempt, while the -relevant gates are held; never infer the result from a later head read. -Direct and served entry points share these semantics. Resource limits constrain -work; their presence must not select a different publication or error protocol. - -Separate known graph publication from optional derived-artifact work. A settled -artifact failure is reported explicitly without relabeling a known commit as -unknown. Genuine uncertain I/O, including a possibly accepted write to a mutable -cache object, still needs the existing settlement and containment rules. Never -discard that uncertainty merely because the data is derived. -Key-scoped storage uncertainty does not itself close process admission; -containment remains limited to the cases defined by the existing storage-outcome -contract, rather than every failure after acceptance. - -Shutdown and successor admission follow the existing -[owned-operation](2026-09-30-owned-server-operations.md) and -[storage-outcome](2026-10-19-storage-outcomes-and-restartable-ownership.md) -contracts. A dropped future does not exclude accepted I/O or a predecessor. -There is no additional startup blocker for a missing maintenance receipt, no -neutral graph commit solely to settle one, and no maintenance-specific unlock. - -### HTTP, receipts and status - -Add `POST /graphs/{graph_id}/optimize` through the shared operation registry, -typed API and [HTTP conventions](2026-10-19-http-api-conventions.md). The graph -is addressed once, in the path. The initial operation uses the existing optimize -behavior without new request options; clients do not copy server ceilings into -a maintenance envelope. -Return the exact completed report after durable publication, or the typed -failure. A no-work result names the unchanged commit. Do not return `202` as a -promise of durable acceptance. - -Here, a receipt means the completed response observed by the caller. It is not -a server-persisted record. The first slice exposes completion through that -response and keeps execution status in the existing owned-operation runtime; -it adds no per-invocation lookup endpoint. A request correlation ID supplies -identification only, not idempotency, deduplication or restart recovery. -Clients and control-plane delivery code must disable automatic POST retries -after ambiguous dispatch. Native status may be lost at restart. - -MCP exposure and asynchronous lookup are deferred. If reconnect lookup becomes -necessary, a separate bounded, process-local extension can retain active and -terminal statuses scoped to the server incarnation. Missing, evicted or old -statuses must mean unknown, never not-executed, and lookup must never execute -work. That extension must define its limits and authorization before shipping; -this RFC does not require a generic jobs API to expose optimize. - -### Bounds and enablement - -Execution is disabled until explicitly configured and qualified. Keep typed -limits under the server's existing configuration/builder ownership, without a -maintenance-only YAML file or parser. The server selects finite ceilings for -tables, fragments, physical rows, source bytes and file references; callers -cannot raise them. Start with one dataset and one compaction task in flight. -Bound metadata enumeration and result construction as well as physical work. - -Validate the complete accepted source inventory before the first rewrite, -including dropped-column materialization, catalog work, table manifests, -index files, deletion files and managed Blob payloads. External Blob references -remain references; their payloads are not fetched. Count before exceeding the -limit, rather than materializing an unbounded inventory and checking afterwards. -Unknown sizes or unsupported layouts refuse. Preserve the current full-text -rebuild boundary; index/statistics/storage errors must not masquerade as -insufficient vector training data. Do not add historical-prefix scans or output -truncation. - -Finite source ceilings and concurrency do not establish hard memory, CPU, -scratch-space or I/O-rate guarantees. Native work and completion resources need -measured limits and process/platform isolation for the supported envelope, -including competition with foreground requests. Existing request-count and -request-byte admission is not a substitute. Ordinary bounded result delivery -remains necessary; the durable-journal report reservation and recovery-envelope -machinery are not part of this contract. - -Managed execution additionally requires native enforcement of control-plane -capacity allocation on every entry point, including direct remote CLI calls. -A proxy check alone is insufficient. State support through explicit deployment -capability and resource admission, not an inference from static-token, signed -token or OIDC authentication. Until qualified, managed execution stays disabled. - -## Invariants - -The [architectural invariants](../dev/invariants.md) remain unchanged: one -accepted snapshot, detached pre-publication effects, one graph publication, -engine authorization, bounded work and typed uncertain outcomes. Derived indexes -and caches affect cost, never logical correctness. Existing ownership and I/O -settlement remain necessary even when observational status is missing. - -There is no second storage writer, manifest-derived job queue, custom WAL, -shadow publication authority, implicit retry, or inline content-write rebuild. -The control plane cannot bypass native policy, resource admission or fencing. -Cleanup, repair, history deletion and isolated workers remain outside this RFC. - -## Compatibility and reversibility - -This design adds no graph format or durable maintenance object. Adding the -`optimize` action requires policy and access-discovery updates for direct and -served callers. Removing the endpoint later leaves ordinary graph commits and -detached output governed by existing retention and cleanup rules. - -The candidate in #1177 has not established a supported wire or disk contract. -Remove its maintenance journal, sequence protocol, lookup/recovery routes, -receipt-only admission blocks and recovery CLI flags before shipping. Do not -silently delete an outstanding journal in an experimental deployment: exclude -its owner and settle its native work using the candidate's recovery procedure -before switching binaries. Migration of such experimental roots is not supplied -by this documentation change. - -The control-plane specifications -[LCY-04.2](https://github.com/ModernRelay/og-control-plane/blob/58a29d6dd4ea00cef84829f03181ffe740d01974/specs/08-LCY-Lifecycle.md#L702) -and [DEC-08-76](https://github.com/ModernRelay/og-control-plane/blob/58a29d6dd4ea00cef84829f03181ffe740d01974/specs/02-DEC-Decisions.md#L3601) -currently require durable native invocation identity, acceptance, results and -original-ID reconciliation. Amend those requirements before claiming managed -compatibility with this weaker observation contract. Moving the same guarantees -into control-plane bookkeeping would not satisfy them. - -## Alternatives - -- **Keep direct-only optimization:** requires stopping the owner for routine - physical maintenance; does not solve the operational problem. -- **Durable native invocation journal:** supports original-ID lookup and - publication reconciliation across crashes, with unresolved or partial outcomes - when evidence is insufficient. It adds persistent identities, retention, - recovery and an availability dependency on receipt settlement. Defer until a - concrete use case requires that stronger guarantee. -- **Reuse the deployment ledger:** couples configuration convergence to physical - optimization without removing durable invocation complexity. -- **Control-plane-only execution receipts:** cannot prove native publication - when a response was lost. Keep workflow history there and preserve unknowns. -- **Process-local asynchronous status:** useful for reconnecting observers, but - adds retention, authorization and incarnation semantics. Defer until the - synchronous slice demonstrates that need. - -## Evidence and tests - -Extend the existing engine `integration/maintenance.rs`, policy, cluster -admission, server operation/runtime and CLI maintenance owners. Prove: - -- Denied actors, unsupported deployments, exhausted bounds and competing work - refuse before physical effects; direct calls cannot bypass engine policy. -- Disconnect and timeout leave execution owned, with no client resubmission. - Shutdown and predecessor exclusion preserve native I/O settlement. -- Success reports this attempt's exact commit; no-work reports the unchanged - commit. Lost responses and crashes never become inferred success or failure. -- Known publication plus settled optional failure preserves both facts; - uncertain publication or optional I/O retains the required native fencing. -- Rows, schema, Blob values, index behavior, retained history and live branches - are preserved. Use GQT for expressible logical assertions; retain Rust owners - for physical mechanisms, concurrency and faults. - -Update the registry, OpenAPI, CLI contracts and DST storage-cost golden with -implementation changes. Verify that no receipt lookup, journal write or -maintenance-specific startup gate remains. Qualify resource and live-cloud -settlement separately; local correctness tests cannot establish those limits. -Measure large Blob rows, wide/high-dimensional rows, many small fragments and -index rebuilds under the intended allocation. Publish the tested limits. - -The candidate's source accounting was inspected against unmodified Lance -13.0.0: [compaction planning/execution](https://docs.rs/crate/lance/13.0.0/source/src/dataset/optimize.rs), -[fragment metadata](https://docs.rs/crate/lance-table/13.0.0/source/src/format/fragment.rs), -[index inventories](https://docs.rs/crate/lance-table/13.0.0/source/src/format/index.rs) -and [object-store sizing](https://docs.rs/crate/lance-io/13.0.0/source/src/object_store.rs). -Lance's compaction byte option alone excludes Blob-v2 payload accounting and -does not bound all optimization work. Reuse qualified native mechanisms rather -than introducing another transaction or recovery protocol. Tests of the prior -journal design are not evidence that this revised contract is implemented. - -## Rollout - -1. Review this reduced contract and align #1177's code, tests, OpenAPI and user - documentation. Keep execution disabled during qualification. -2. Enable the synchronous self-hosted slice only for the measured supported - resource envelope. Preserve explicit unknown outcomes and existing shutdown - behavior; publish the policy change and limits with the implementation. -3. Amend the control-plane specifications and qualify native capacity admission - before enabling managed calls. Keep scheduling and operational history in - the control plane. This phase does not gate the self-hosted slice. - -## Unresolved questions - -No additional design decision is required for the synchronous scope. The -implementation, resource and managed-integration evidence gates are listed in -`blocked_on`. Durable receipts, asynchronous lookup and worker offload require -separate justification and are not implied follow-on commitments. - -## Decision log - -- **2026-10-27:** Narrow the initial candidate to safe execution through the - existing owner. Replace durable invocation/lookup and offline recovery with - synchronous exact results and explicit unknown outcomes. Assign orchestration, - history and log collection to the control plane; retain native authorization, - resource admission, publication and settlement. Record the control-plane - specification changes needed before managed enablement. The implementation - still needs to be aligned; this revision does not accept its earlier protocol. diff --git a/docs/rfcs/README.md b/docs/rfcs/README.md index 60c56421b..5b8afb7a7 100644 --- a/docs/rfcs/README.md +++ b/docs/rfcs/README.md @@ -254,4 +254,3 @@ then dated RFCs by date. | [2026-10-19](2026-10-19-http-api-conventions.md) | HTTP API conventions | maintainer | accepted | complete | | [2026-10-19](2026-10-19-storage-outcomes-and-restartable-ownership.md) | Storage outcomes per operation and restartable ownership | maintainer | accepted | in-progress | | [2026-10-20](2026-10-20-operator-connections-and-credential-providers.md) | Operator profiles and credential providers | maintainer | draft | in-progress | -| [2026-10-25](2026-10-25-served-optimization.md) | Served optimization | maintainer | draft | in-progress |