diff --git a/sandbox/libs/analytics-api/src/main/java/org/opensearch/analytics/schema/IpType.java b/sandbox/libs/analytics-api/src/main/java/org/opensearch/analytics/schema/IpType.java index e8ee095fba280..d8a0ce138d9f2 100644 --- a/sandbox/libs/analytics-api/src/main/java/org/opensearch/analytics/schema/IpType.java +++ b/sandbox/libs/analytics-api/src/main/java/org/opensearch/analytics/schema/IpType.java @@ -12,7 +12,6 @@ import org.apache.calcite.sql.type.SqlTypeName; import java.util.Locale; - /** * Calcite type marker for an OpenSearch {@code ip} column. Backed by * {@link SqlTypeName#VARBINARY} so planner coercion is unchanged; the subclass exists diff --git a/sandbox/libs/dataformat-native/rust/Cargo.toml b/sandbox/libs/dataformat-native/rust/Cargo.toml index b5f7cb0440f92..69d9185a3258a 100644 --- a/sandbox/libs/dataformat-native/rust/Cargo.toml +++ b/sandbox/libs/dataformat-native/rust/Cargo.toml @@ -84,8 +84,12 @@ opensearch-repository-azure = { path = "../../../plugins/native-repository-azure opensearch-repository-fs = { path = "../../../plugins/native-repository-fs/src/main/rust" } [profile.release] -lto = true -codegen-units = 1 +# POC iteration speed: LTO off + parallel codegen. Fat LTO with +# codegen-units=1 is the *benchmarking* configuration — restore it +# (lto = true, codegen-units = 1) before taking performance numbers; +# expect noticeably slower query/fold paths without it. +lto = false +codegen-units = 16 incremental = true debug = "line-tables-only" strip = false diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs index c884828e71791..1c86ddc35e89e 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/api.rs @@ -924,7 +924,8 @@ pub async unsafe fn execute_query( let shard_view = &*(shard_view_ptr as *const ShardView); let runtime = &*(runtime_ptr as *const DataFusionRuntime); let cpu_executor = manager.cpu_executor(); - + // a > 5 && name = 'arpit' --> ? + // substrait a > 5 && (indexed_filter) // Create per-query context (auto-registers in the global registry) and extract // its per-query memory pool overlaying the global pool. let global_pool = runtime.runtime_env.memory_pool.clone(); diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs index 60864b563014b..3fe3a7b015352 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs @@ -579,6 +579,232 @@ pub unsafe extern "C" fn df_query_registry_top_n_by_current( Ok(written as i64) } +/// POC(mv) v2 search: Final-fold via a caller-provided SQL template +/// (placeholder __MV_STATES__ replaced with the UNION ALL of state files). +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_mv_search_v2( + files_ptr: *const *const u8, + files_lens: *const i64, + files_count: i64, + sql_ptr: *const u8, + sql_len: i64, + out_ptr: *mut u8, + out_cap: i64, + out_len: *mut i64, +) -> i64 { + let mut files = Vec::with_capacity(files_count as usize); + for i in 0..files_count as usize { + let f_ptr = *files_ptr.add(i); + let f_len = *files_lens.add(i); + files.push( + str_from_raw(f_ptr, f_len) + .map_err(|e| format!("df_mv_search_v2: file[{}]: {}", i, e))? + .to_string(), + ); + } + let sql = str_from_raw(sql_ptr, sql_len).map_err(|e| format!("df_mv_search_v2: sql: {}", e))?; + let text = crate::mv_writer::mv_search_v2(&files, sql)?; + write_out_buffer( + text.as_bytes(), + out_ptr, + out_cap, + out_len, + "mv search v2 result", + )?; + Ok(0) +} + +/// POC(mv) streaming writer lifecycle — create/feed/finalize/abort. +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_mv_writer_create( + sql_ptr: *const u8, + sql_len: i64, + num_group_cols: i64, +) -> i64 { + let sql = + str_from_raw(sql_ptr, sql_len).map_err(|e| format!("df_mv_writer_create: sql: {}", e))?; + Ok(crate::mv_writer::mv_writer_create(sql, num_group_cols)) +} + +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_mv_writer_feed(writer_id: i64, array_ptr: i64, schema_ptr: i64) -> i64 { + use arrow_array::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; + let ffi_array = FFI_ArrowArray::from_raw(array_ptr as *mut FFI_ArrowArray); + let ffi_schema = FFI_ArrowSchema::from_raw(schema_ptr as *mut FFI_ArrowSchema); + let mut array_data = arrow_array::ffi::from_ffi(ffi_array, &ffi_schema) + .map_err(|e| format!("df_mv_writer_feed: import: {}", e))?; + array_data.align_buffers(); + let struct_array = arrow_array::StructArray::from(array_data); + let batch = arrow_array::RecordBatch::from(struct_array); + crate::mv_writer::mv_writer_feed(writer_id, &batch)?; + Ok(0) +} + +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_mv_writer_finalize( + writer_id: i64, + out_ptr: *const u8, + out_len: i64, +) -> i64 { + let output = + str_from_raw(out_ptr, out_len).map_err(|e| format!("df_mv_writer_finalize: out: {}", e))?; + let rows = crate::mv_writer::mv_writer_finalize(writer_id, output)?; + Ok(rows) +} + +/// Finalizes the MV writer and exports the sorted state batch via Arrow +/// C-Data into caller-allocated `FFI_ArrowArray`/`FFI_ArrowSchema` structs +/// (zero copy — the JVM imports the same buffers; Rust's release callback +/// frees them when the consumer closes). Returns the state row count. +/// +/// # Safety +/// `array_addr` / `schema_addr` must point to caller-allocated, uninitialized +/// ArrowArray / ArrowSchema C structs that the caller will import exactly once. +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_mv_writer_finalize_arrow( + writer_id: i64, + array_addr: i64, + schema_addr: i64, +) -> i64 { + crate::mv_writer::mv_writer_finalize_arrow(writer_id, array_addr, schema_addr) +} + +#[no_mangle] +pub unsafe extern "C" fn df_mv_writer_abort(writer_id: i64) { + crate::mv_writer::mv_writer_abort(writer_id); +} + +/// POC(mv): Final-aggregate over MV state files. Writes result text (svc\tcount lines) +/// into the caller buffer via write_out_buffer. +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_mv_search_poc( + files_ptr: *const *const u8, + files_lens: *const i64, + files_count: i64, + group_ptr: *const u8, + group_len: i64, + state_ptr: *const u8, + state_len: i64, + out_ptr: *mut u8, + out_cap: i64, + out_len: *mut i64, +) -> i64 { + let mut files = Vec::with_capacity(files_count as usize); + for i in 0..files_count as usize { + let f_ptr = *files_ptr.add(i); + let f_len = *files_lens.add(i); + files.push( + str_from_raw(f_ptr, f_len) + .map_err(|e| format!("df_mv_search_poc: file[{}]: {}", i, e))? + .to_string(), + ); + } + let group = str_from_raw(group_ptr, group_len) + .map_err(|e| format!("df_mv_search_poc: group: {}", e))?; + let state = str_from_raw(state_ptr, state_len) + .map_err(|e| format!("df_mv_search_poc: state: {}", e))?; + let text = crate::mv_poc::mv_search_poc(&files, group, state)?; + write_out_buffer( + text.as_bytes(), + out_ptr, + out_cap, + out_len, + "mv search result", + )?; + Ok(0) +} + +/// POC(mv): build the hardcoded MV state file from a primary parquet file. +/// Blocking; returns state-row count (>=0) on success. +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_mv_build_poc( + input_ptr: *const u8, + input_len: i64, + table_ptr: *const u8, + table_len: i64, + sql_ptr: *const u8, + sql_len: i64, + output_ptr: *const u8, + output_len: i64, +) -> i64 { + let input = + str_from_raw(input_ptr, input_len).map_err(|e| format!("df_mv_build_poc: input: {}", e))?; + let table = + str_from_raw(table_ptr, table_len).map_err(|e| format!("df_mv_build_poc: table: {}", e))?; + let sql = str_from_raw(sql_ptr, sql_len).map_err(|e| format!("df_mv_build_poc: sql: {}", e))?; + let output = str_from_raw(output_ptr, output_len) + .map_err(|e| format!("df_mv_build_poc: output: {}", e))?; + let rows = crate::mv_poc::mv_build_poc(input, table, sql, output)?; + Ok(rows) +} + +/// Refresh-time ship build: run the definition's Partial over one parquet +/// file and export the sorted state batch via Arrow C-Data (no scratch file). +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_session_attach_mv( + handle_ptr: i64, + mv_paths_ptr: *const u8, + mv_paths_len: i64, + covered_names_ptr: *const u8, + covered_names_len: i64, + strict: u8, +) -> i64 { + let handle = &mut *(handle_ptr as *mut crate::session_context::SessionContextHandle); + let mv_paths = str_from_raw(mv_paths_ptr, mv_paths_len) + .map_err(|e| format!("df_session_attach_mv: mv_paths: {}", e))?; + let covered_names = str_from_raw(covered_names_ptr, covered_names_len) + .map_err(|e| format!("df_session_attach_mv: covered_names: {}", e))?; + let mv_file_paths: Vec = mv_paths + .split('\n') + .filter(|s| !s.is_empty()) + .map(str::to_string) + .collect(); + let covered_raw_file_names: std::collections::HashSet = covered_names + .split('\n') + .filter(|s| !s.is_empty()) + .map(str::to_string) + .collect(); + if mv_file_paths.is_empty() { + return Ok(0); // nothing covered — leave the session raw-only + } + handle.mv_binding = Some(crate::mv_read::MVBinding { + mv_file_paths, + covered_raw_file_names, + strict: strict != 0, + }); + Ok(0) +} + +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_mv_build_arrow( + input_ptr: *const u8, + input_len: i64, + table_ptr: *const u8, + table_len: i64, + sql_ptr: *const u8, + sql_len: i64, + array_addr: i64, + schema_addr: i64, +) -> i64 { + let input = str_from_raw(input_ptr, input_len) + .map_err(|e| format!("df_mv_build_arrow: input: {}", e))?; + let table = str_from_raw(table_ptr, table_len) + .map_err(|e| format!("df_mv_build_arrow: table: {}", e))?; + let sql = + str_from_raw(sql_ptr, sql_len).map_err(|e| format!("df_mv_build_arrow: sql: {}", e))?; + let rows = crate::mv_poc::mv_build_arrow(input, table, sql, array_addr, schema_addr)?; + Ok(rows) +} + #[ffm_safe] #[no_mangle] pub unsafe extern "C" fn df_sql_to_substrait( diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs index 4d5e81f9f7cf5..1deda3c1e8590 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_executor.rs @@ -145,6 +145,7 @@ pub async fn execute_indexed_query( has_topk: false, prepared_plan: None, phantom_reservation: None, + mv_binding: None, }; let ptr = Box::into_raw(Box::new(handle)) as i64; @@ -1431,6 +1432,13 @@ async unsafe fn execute_indexed_with_context_inner( } else { physical_plan }; + // MV read path (indexed executor = the live shard fragment path on this + // branch): a session binding replaces the Partial with the mv-state file + // scan (STRICT — misalignment throws, never a silent fallback). + let physical_plan = match handle.mv_binding.as_ref() { + Some(binding) => crate::mv_read::apply_mv_binding(&ctx, physical_plan, binding).await?, + None => physical_plan, + }; let target_schema = crate::schema_coerce::coerce_inferred_schema(physical_plan.schema()); let physical_plan = crate::relabel_exec::wrap_if_relabel_needed(physical_plan, target_schema)?; log_debug!( diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/stream.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/stream.rs index 399ebeb1d0455..746a95e9cc48a 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/stream.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/indexed_table/stream.rs @@ -853,7 +853,7 @@ impl IndexedStream { impl Stream for IndexedStream { type Item = Result; - + // 10 > A > 5 OR Text = 'Arpit' fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { // Manual timer for `elapsed_compute`: total wall time spent // inside this poll. Attributed to the operator for EXPLAIN diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs index 8708627abe3a4..daf3d1a170c21 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs @@ -34,6 +34,9 @@ pub mod indexed_table; pub mod local_executor; pub mod memory; pub mod memory_guard; +pub mod mv_poc; +pub mod mv_read; +pub mod mv_writer; pub mod native_error; pub mod partition_stream; pub mod patterns; diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/mv_poc.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/mv_poc.rs new file mode 100644 index 0000000000000..e28ead88f4652 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/mv_poc.rs @@ -0,0 +1,202 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! POC(mv): self-contained materialized-view build. +//! +//! One blocking call: read a primary parquet file, run the hardcoded MV query +//! stopped at Partial mode, write the state batches to an MV parquet file. +//! Used by the mv-data-format plugin's writer at flush time. +//! +//! Deliberately independent of the runtime manager / ShardView machinery: a +//! private current-thread tokio runtime per call. This is POC-grade — the +//! production build path goes through the engine session + memory pools. + +use std::fs::File; +use std::sync::Arc; + +use datafusion::physical_plan::aggregates::{AggregateExec, AggregateMode}; +use datafusion::physical_plan::{collect, ExecutionPlan}; +use datafusion::prelude::{ParquetReadOptions, SessionContext}; +use parquet::arrow::ArrowWriter; + +/// Shared core: run the definition's Partial stage over one parquet file and +/// sort the resulting state rows by the group key (col 0) so downstream merges +/// are streaming k-way folds (memory ∝ cursors, not groups). Sort cost ∝ +/// #groups, never #docs. +fn build_sorted_state( + input_file: &str, + table_name: &str, + sql: &str, +) -> Result { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|e| format!("mv_poc runtime: {e}"))?; + + rt.block_on(async { + let ctx = SessionContext::new(); + ctx.register_parquet(table_name, input_file, ParquetReadOptions::default()) + .await + .map_err(|e| format!("mv_poc register_parquet({input_file}): {e}"))?; + + let df = ctx + .sql(sql) + .await + .map_err(|e| format!("mv_poc plan sql: {e}"))?; + let physical = df + .create_physical_plan() + .await + .map_err(|e| format!("mv_poc physical plan: {e}"))?; + let partial = find_partial(&physical) + .ok_or_else(|| "mv_poc: no Partial aggregate in plan".to_string())?; + + let batches = collect(partial, ctx.task_ctx()) + .await + .map_err(|e| format!("mv_poc collect partial: {e}"))?; + + // Schema comes from the plan even when batches are empty — callers + // rely on a well-formed (possibly zero-row) state batch. + let schema = if batches.is_empty() { + return Err( + "mv_poc: partial produced no batches (expected at least an empty batch)" + .to_string(), + ); + } else { + batches[0].schema() + }; + + let concatenated = arrow::compute::concat_batches(&schema, &batches) + .map_err(|e| format!("mv_poc concat: {e}"))?; + let sort_indices = arrow::compute::sort_to_indices(concatenated.column(0), None, None) + .map_err(|e| format!("mv_poc sort: {e}"))?; + let sorted_columns: Result, _> = concatenated + .columns() + .iter() + .map(|c| arrow::compute::take(c.as_ref(), &sort_indices, None)) + .collect(); + arrow_array::RecordBatch::try_new( + schema, + sorted_columns.map_err(|e| format!("mv_poc take: {e}"))?, + ) + .map_err(|e| format!("mv_poc sorted batch: {e}")) + }) +} + +/// Builds the MV state file (Arrow IPC, decision 17 — the fold readers use +/// register_arrow). Returns the number of state rows written. +pub fn mv_build_poc( + input_file: &str, + table_name: &str, + sql: &str, + output_file: &str, +) -> Result { + let sorted = build_sorted_state(input_file, table_name, sql)?; + let file = + File::create(output_file).map_err(|e| format!("mv_poc create {output_file}: {e}"))?; + let mut writer = arrow::ipc::writer::FileWriter::try_new(file, &sorted.schema()) + .map_err(|e| format!("mv_poc ipc writer: {e}"))?; + writer + .write(&sorted) + .map_err(|e| format!("mv_poc write batch: {e}"))?; + writer.finish().map_err(|e| format!("mv_poc finish: {e}"))?; + Ok(sorted.num_rows() as i64) +} + +/// Refresh-time build for the ship path: same Partial build + group-key sort +/// as [`mv_build_poc`], but the sorted state batch is EXPORTED via Arrow +/// C-Data into caller-provided struct addresses (zero copy into the JVM) +/// instead of being written to a file. Returns the state row count. +pub fn mv_build_arrow( + input_file: &str, + table_name: &str, + sql: &str, + array_addr: i64, + schema_addr: i64, +) -> Result { + use arrow_array::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; + use arrow_array::Array; // into_data + let sorted = build_sorted_state(input_file, table_name, sql)?; + let rows = sorted.num_rows() as i64; + let struct_array: arrow_array::StructArray = sorted.into(); + let data = struct_array.into_data(); + let ffi_schema = FFI_ArrowSchema::try_from(data.data_type()) + .map_err(|e| format!("mv_build_arrow schema export: {e}"))?; + let ffi_array = FFI_ArrowArray::new(&data); + unsafe { + std::ptr::write(array_addr as *mut FFI_ArrowArray, ffi_array); + std::ptr::write(schema_addr as *mut FFI_ArrowSchema, ffi_schema); + } + Ok(rows) +} + +fn find_partial(plan: &Arc) -> Option> { + if let Some(agg) = plan.downcast_ref::() { + // Partial: the state-producing stage of a split plan. Single / + // SinglePartitioned: small inputs (e.g. one tiny parquet) skip the + // split — output schema EQUALS the Partial state schema because MV + // definitions are pre-decomposed to their mergeable core (SUM/COUNT/ + // MIN/MAX only; no raw AVG), so accepting it is exact, not a fallback. + match agg.mode() { + AggregateMode::Partial | AggregateMode::Single | AggregateMode::SinglePartitioned => { + return Some(plan.clone()); + } + _ => {} + } + } + for child in plan.children() { + if let Some(found) = find_partial(child) { + return Some(found); + } + } + None +} + +/// POC(mv) search: Final-style aggregation over MV state files. Returns rows +/// as "service\tcount" lines joined by newlines (POC-grade wire format). +pub fn mv_search_poc( + state_files: &[String], + group_key: &str, + state_col: &str, +) -> Result { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|e| format!("mv_poc search runtime: {e}"))?; + + rt.block_on(async { + let ctx = SessionContext::new(); + for (i, f) in state_files.iter().enumerate() { + ctx.register_parquet(&format!("mv_{i}"), f.as_str(), ParquetReadOptions::default()) + .await + .map_err(|e| format!("mv_poc search register {f}: {e}"))?; + } + let union_sql = (0..state_files.len()) + .map(|i| format!("SELECT * FROM mv_{i}")) + .collect::>() + .join(" UNION ALL "); + let sql = format!( + "SELECT \"{group_key}\", SUM(\"{state_col}\") AS cnt FROM ({union_sql}) GROUP BY \"{group_key}\" ORDER BY \"{group_key}\"" + ); + let df = ctx.sql(&sql).await.map_err(|e| format!("mv_poc search sql: {e}"))?; + let batches = df.collect().await.map_err(|e| format!("mv_poc search collect: {e}"))?; + + let mut out = String::new(); + for b in &batches { + let svc = arrow::compute::cast(b.column(0), &arrow_schema::DataType::Utf8) + .map_err(|e| format!("cast svc: {e}"))?; + let cnt = arrow::compute::cast(b.column(1), &arrow_schema::DataType::Int64) + .map_err(|e| format!("cast cnt: {e}"))?; + let svc = svc.as_any().downcast_ref::().ok_or("svc downcast")?; + let cnt = cnt.as_any().downcast_ref::().ok_or("cnt downcast")?; + for i in 0..b.num_rows() { + out.push_str(&format!("{}\t{}\n", svc.value(i), cnt.value(i))); + } + } + Ok(out) + }) +} diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/mv_read.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/mv_read.rs new file mode 100644 index 0000000000000..8805b37333536 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/mv_read.rs @@ -0,0 +1,275 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! Materialized-view read path: prepare-time plan surgery. +//! +//! When a session carries an [`MVBinding`] (attached by the Java shard-scan +//! handler from the catalog snapshot's coverage), `prepare_partial_plan` +//! calls [`apply_mv_binding`] on the stripped Partial plan: +//! +//! ```text +//! AggregateExec(Partial) UnionExec +//! │ ──▶ ╱ ╲ +//! scan(all raw files) scan(mv state AggregateExec(Partial) +//! files, aliased) │ +//! scan(uncovered raw files) +//! ``` +//! +//! MV state files ARE Partial-mode output (the zero-translation contract), +//! so the union's two branches produce the same shape and the coordinator +//! FINAL consumes the union unchanged. +//! +//! Every step is fallback-first: any mismatch (schema shape, plan shape, +//! scan-leaf shape) returns the original plan untouched — the query is then +//! answered entirely from raw parquet. Never wrong, only slower. +//! +//! ## Strict (MV-only) mode — POC verification +//! +//! With [`MVBinding::strict`] set, the contract inverts to fail-closed: +//! every fallback path becomes a hard error, every raw file must be covered, +//! and the produced plan is the aliased state-file scan ONLY — no raw scan +//! node exists in the plan, so the query physically cannot read raw parquet. +//! A successful strict query is therefore proof the answer came exclusively +//! from MV state files. POC/testing only; never enable in production paths. + +use std::collections::HashSet; +use std::sync::Arc; + +use datafusion::arrow::datatypes::SchemaRef; +use datafusion::catalog::TableProvider; +use datafusion::common::Result; +use datafusion::datasource::file_format::arrow::ArrowFormat; +use datafusion::datasource::listing::{ + ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl, +}; +use datafusion::datasource::source::DataSourceExec; +use datafusion::physical_plan::aggregates::{AggregateExec, AggregateMode}; +use datafusion::physical_plan::expressions::Column; +use datafusion::physical_plan::projection::ProjectionExec; +use datafusion::physical_plan::union::UnionExec; +use datafusion::physical_plan::{ExecutionPlan, PhysicalExpr}; +use datafusion::prelude::SessionContext; +use datafusion_datasource::file_groups::FileGroup; +use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder}; +use native_bridge_common::{log_debug, log_error, log_info}; + +/// Shard-local MV coverage, attached via `df_session_attach_mv`. +pub struct MVBinding { + /// Absolute paths of MV state parquet files (covered segments). + pub mv_file_paths: Vec, + /// Raw parquet file NAMES (last path segment) of covered segments — + /// excluded from the raw branch when the MV branch is taken. + pub covered_raw_file_names: HashSet, + /// POC MV-only verification mode: fallbacks become hard errors and the + /// plan must be servable from state files alone (see module docs). + pub strict: bool, +} + +/// Rewrites a stripped Partial plan into UNION(mv scan, Partial over uncovered). +/// Returns the input plan unchanged when the binding can't be applied safely. +pub async fn apply_mv_binding( + ctx: &SessionContext, + stripped: Arc, + binding: &MVBinding, +) -> Result> { + match try_apply(ctx, Arc::clone(&stripped), binding).await { + Ok(Some(plan)) => Ok(plan), + Ok(None) if binding.strict => Err(datafusion::common::DataFusionError::Execution( + "mv_read strict: binding did not apply (fallback path taken); \ + see preceding mv_read log for the reason" + .to_string(), + )), + Ok(None) => Ok(stripped), + Err(e) if binding.strict => Err(e), + Err(e) => { + log_error!("mv_read: binding failed, falling back to raw plan: {}", e); + Ok(stripped) + } + } +} + +async fn try_apply( + ctx: &SessionContext, + stripped: Arc, + binding: &MVBinding, +) -> Result>> { + if binding.mv_file_paths.is_empty() { + return Ok(None); + } + // v0 scope: the stripped plan's root must be the Partial aggregate itself. + // TopK fragments (Sort/Fetch above the Partial) and any other wrapper are + // out of scope — fallback. (RelabelExec wrapping happens AFTER this + // function; see prepare_partial_plan ordering.) + let Some(root_agg) = stripped.downcast_ref::() else { + log_debug!("mv_read: plan root is not AggregateExec — fallback"); + return Ok(None); + }; + if *root_agg.mode() != AggregateMode::Partial { + log_debug!("mv_read: root aggregate is not Partial — fallback"); + return Ok(None); + } + + // Build the MV state scan. + let mv_scan = build_mv_scan(ctx, &binding.mv_file_paths).await?; + + // Positional schema alignment (the state contract is positional: group-by + // columns first, then state columns). Column NAMES differ — the writer used + // its own table alias — so names are aliased to the Partial output's names. + // Field count or type mismatch = state files don't match this query = fallback. + let partial_schema = stripped.schema(); + let mv_schema = mv_scan.schema(); + if !schemas_align(&partial_schema, &mv_schema) { + log_error!( + "mv_read: state schema does not align with partial output (partial={}, mv={}) — fallback", + partial_schema, + mv_schema + ); + return Ok(None); + } + let aliased_mv = alias_positionally(mv_scan, &partial_schema)?; + + // Narrow the raw branch to uncovered files. If the scan leaf can't be + // rewritten (unexpected plan shape), fall back — running covered segments + // through BOTH branches would double-count. + let Some((narrowed, remaining_raw_files)) = + narrow_scan_files(Arc::clone(&stripped), &binding.covered_raw_file_names)? + else { + log_debug!("mv_read: could not narrow raw scan — fallback"); + return Ok(None); + }; + + if binding.strict { + // MV-only: every raw file must be covered, and the plan is the state + // scan alone — no raw scan node exists, raw parquet cannot be read. + if remaining_raw_files > 0 { + return Err(datafusion::common::DataFusionError::Execution(format!( + "mv_read strict: {} raw file(s) not covered by MV state — \ + query cannot be served from the MV alone", + remaining_raw_files + ))); + } + log_info!( + "mv_read strict: serving from {} state files ONLY (no raw scan in plan)", + binding.mv_file_paths.len() + ); + return Ok(Some(aliased_mv)); + } + + log_info!( + "mv_read: bound {} state files; raw branch narrowed by {} covered files", + binding.mv_file_paths.len(), + binding.covered_raw_file_names.len() + ); + Ok(Some(UnionExec::try_new(vec![aliased_mv, narrowed])?)) +} + +/// Builds a plan scanning the MV state files (Arrow IPC, decision 17). +async fn build_mv_scan( + ctx: &SessionContext, + mv_file_paths: &[String], +) -> Result> { + let urls: Vec = mv_file_paths + .iter() + .map(ListingTableUrl::parse) + .collect::>()?; + let listing_options = ListingOptions::new(Arc::new(ArrowFormat)).with_file_extension(".arrow"); + let schema = listing_options.infer_schema(&ctx.state(), &urls[0]).await?; + let config = ListingTableConfig::new_with_multi_paths(urls) + .with_listing_options(listing_options) + .with_schema(schema); + let table = ListingTable::try_new(config)?; + table.scan(&ctx.state(), None, &[], None).await +} + +/// Positional alignment check: equal field count and bit-compatible types. +/// Names deliberately NOT compared (writer alias differs from query alias). +fn schemas_align(partial: &SchemaRef, mv: &SchemaRef) -> bool { + if partial.fields().len() != mv.fields().len() { + return false; + } + partial + .fields() + .iter() + .zip(mv.fields().iter()) + .all(|(p, m)| p.data_type() == m.data_type()) +} + +/// Wraps `input` in a ProjectionExec that renames columns positionally to +/// `target` field names, so UnionExec sees identical schemas on both branches. +fn alias_positionally( + input: Arc, + target: &SchemaRef, +) -> Result> { + let exprs: Vec<(Arc, String)> = target + .fields() + .iter() + .enumerate() + .map(|(i, f)| { + ( + Arc::new(Column::new(input.schema().field(i).name(), i)) as Arc, + f.name().clone(), + ) + }) + .collect(); + Ok(Arc::new(ProjectionExec::try_new(exprs, input)?)) +} + +/// Rewrites the plan's single parquet scan leaf to exclude `covered` file names +/// (matched on the last path segment). Returns None when the plan doesn't have +/// the expected shape (single-child wrappers down to one DataSourceExec leaf). +fn narrow_scan_files( + plan: Arc, + covered: &HashSet, +) -> Result, usize)>> { + if let Some(source_exec) = plan.downcast_ref::() { + let Some(file_config) = source_exec + .data_source() + .as_ref() + .downcast_ref::() + else { + return Ok(None); + }; + let mut remaining = 0usize; + let narrowed_groups: Vec = file_config + .file_groups + .iter() + .map(|group| { + FileGroup::new( + group + .iter() + .filter(|pf| { + let name = pf.object_meta.location.filename().unwrap_or_default(); + let keep = !covered.contains(name); + if keep { + remaining += 1; + } + keep + }) + .cloned() + .collect(), + ) + }) + .collect(); + let narrowed_config = FileScanConfigBuilder::from(file_config.clone()) + .with_file_groups(narrowed_groups) + .build(); + let narrowed: Arc = DataSourceExec::from_data_source(narrowed_config); + return Ok(Some((narrowed, remaining))); + } + // Recurse through single-child wrappers; bail on multi-input nodes. + let children = plan.children(); + if children.len() != 1 { + return Ok(None); + } + match narrow_scan_files(Arc::clone(children[0]), covered)? { + Some((new_child, remaining)) => { + Ok(Some((plan.with_new_children(vec![new_child])?, remaining))) + } + None => Ok(None), + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/mv_writer.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/mv_writer.rs new file mode 100644 index 0000000000000..97f2073d6ea5b --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/mv_writer.rs @@ -0,0 +1,401 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! POC(mv) v2: streaming MV writer with DataFusion-maintained state. +//! +//! Lifecycle (mirrors the parquet native writer): `create(definition SQL)` / +//! `feed(batch)` / `finalize(path)` / `abort`. +//! +//! Architecture (decided 2026-08-03): **hash aggregation until flush + one +//! sort at flush.** +//! - `feed`: run the definition's aggregation STOPPED AT `Partial` mode over +//! the fed batch (raw rows → state rows), then fold the resulting mini +//! state batch into the held state via `PartialReduce` (state ⊕ state) — +//! DataFusion owns the entire state algebra; nothing is hand-rolled. +//! - `finalize`: drain the held state → single sort by group keys (cost ∝ +//! groups, never docs) → write the sorted state parquet. +//! +//! POC simplification: the held state is a Vec of state batches compacted via +//! PartialReduce whenever it grows past a threshold, rather than one live +//! GroupedHashAggregateStream fed by a channel. Same operator algebra, no +//! long-lived task per writer. +//! +//! TODO(mv, noted optimization — do not build yet): for high group +//! cardinality, replace hash-until-flush with sorted runs per rotation + +//! `SortPreservingMergeExec` + order-aware (`GroupOrdered`) fold at finalize: +//! spillable runs, O(1)-group fold memory. Trigger: all-groups hash state +//! memory ∝ G hurts. See mv-incremental-lld §3.4 / KB notes. + +use std::fs::File; +use std::sync::atomic::{AtomicI64, Ordering}; +use std::sync::{Arc, Mutex}; + +use arrow_array::{Array, RecordBatch}; +use arrow_schema::Schema; +use datafusion::catalog::TableProvider; +use datafusion::datasource::MemTable; +use datafusion::physical_plan::aggregates::{AggregateExec, AggregateMode}; +use datafusion::physical_plan::{collect, ExecutionPlan}; +use datafusion::prelude::{ParquetReadOptions, SessionContext}; +use parquet::arrow::ArrowWriter; + +/// Compact the held state batches via PartialReduce when the count exceeds this. +const COMPACT_THRESHOLD: usize = 8; + +/// One live MV writer. +struct MvWriterState { + /// The MV definition, e.g. "SELECT service, status, COUNT(*), ... FROM mv_input GROUP BY service, status". + /// Table name inside the SQL must be `mv_input`. + sql: String, + /// Raw-input schema (captured from the first fed batch) for planning. + input_schema: Option>, + /// Accumulated partial-state batches (each already state rows). + state_batches: Vec, + /// Number of group-by columns (leading columns of the state schema). + num_group_cols: usize, +} + +pub struct MvWriterHandle { + state: Mutex, +} + +static NEXT_ID: AtomicI64 = AtomicI64::new(1); + +static WRITERS: std::sync::LazyLock>>> = + std::sync::LazyLock::new(|| Mutex::new(std::collections::HashMap::new())); + +pub fn mv_writer_create(sql: &str, num_group_cols: i64) -> i64 { + let id = NEXT_ID.fetch_add(1, Ordering::Relaxed); + WRITERS.lock().unwrap().insert( + id, + Arc::new(MvWriterHandle { + state: Mutex::new(MvWriterState { + sql: sql.to_string(), + input_schema: None, + state_batches: Vec::new(), + num_group_cols: num_group_cols as usize, + }), + }), + ); + id +} + +fn get_writer(id: i64) -> Result, String> { + WRITERS + .lock() + .unwrap() + .get(&id) + .cloned() + .ok_or_else(|| format!("mv_writer: unknown handle {id}")) +} + +fn find_agg_with_mode( + plan: &Arc, + mode: AggregateMode, +) -> Option> { + if let Some(agg) = plan.downcast_ref::() { + if *agg.mode() == mode { + return Some(plan.clone()); + } + } + for child in plan.children() { + if let Some(found) = find_agg_with_mode(child, mode) { + return Some(found); + } + } + None +} + +fn block_on(fut: F) -> Result { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|e| format!("mv_writer runtime: {e}"))?; + Ok(rt.block_on(fut)) +} + +/// Runs the definition SQL over `mv_input` = the given batches, returning the +/// PARTIAL-mode output (state rows). DataFusion's Partial operator does the +/// raw→state lift; the same helper serves feed (raw input) because Partial +/// over raw rows IS the fold for a single batch. +fn partial_states_over( + sql: &str, + schema: Arc, + batches: Vec, +) -> Result, String> { + block_on(async move { + // Build the session WITHOUT CombinePartialFinalAggregate: on small + // single-partition inputs that rule collapses the Partial/Final pair + // into Single mode and the Partial node this writer depends on + // disappears ("no Partial aggregate in plan" — the near-empty-batch + // flush failure). Same rule removal the engine applies on every + // execution path (agg_mode.rs). + let state = datafusion::execution::session_state::SessionStateBuilder::new() + .with_default_features() + .with_physical_optimizer_rules( + crate::agg_mode::physical_optimizer_rules_without_combine(), + ) + .build(); + let ctx = SessionContext::new_with_state(state); + let mem = MemTable::try_new(schema, vec![batches]).map_err(|e| format!("memtable: {e}"))?; + ctx.register_table("mv_input", Arc::new(mem)) + .map_err(|e| format!("register: {e}"))?; + let df = ctx.sql(sql).await.map_err(|e| format!("plan: {e}"))?; + let physical = df + .create_physical_plan() + .await + .map_err(|e| format!("physical: {e}"))?; + let partial = find_agg_with_mode(&physical, AggregateMode::Partial) + .ok_or("no Partial aggregate in plan")?; + collect(partial, ctx.task_ctx()) + .await + .map_err(|e| format!("collect: {e}")) + })? +} + +/// Folds N state batches into compacted state batches via PartialReduce: +/// state ⊕ state → state. Constructed the same way agg_mode.rs builds +/// PartialReduce nodes — by planning the definition and swapping the +/// Partial node's mode, with the state batches as input. +fn partial_reduce( + sql: &str, + input_schema: Arc, + state_batches: Vec, +) -> Result, String> { + if state_batches.len() <= 1 { + return Ok(state_batches); + } + let state_schema = state_batches[0].schema(); + block_on(async move { + // Build the session WITHOUT CombinePartialFinalAggregate: on small + // single-partition inputs that rule collapses the Partial/Final pair + // into Single mode and the Partial node this writer depends on + // disappears ("no Partial aggregate in plan" — the near-empty-batch + // flush failure). Same rule removal the engine applies on every + // execution path (agg_mode.rs). + let state = datafusion::execution::session_state::SessionStateBuilder::new() + .with_default_features() + .with_physical_optimizer_rules( + crate::agg_mode::physical_optimizer_rules_without_combine(), + ) + .build(); + let ctx = SessionContext::new_with_state(state); + // Plan the definition over a dummy raw table to obtain the aggregate + // node shape (group exprs + aggregate exprs + schemas)... + let dummy = + MemTable::try_new(input_schema, vec![vec![]]).map_err(|e| format!("dummy: {e}"))?; + ctx.register_table("mv_input", Arc::new(dummy)) + .map_err(|e| format!("register: {e}"))?; + let df = ctx.sql(sql).await.map_err(|e| format!("plan: {e}"))?; + let physical = df + .create_physical_plan() + .await + .map_err(|e| format!("physical: {e}"))?; + let partial = find_agg_with_mode(&physical, AggregateMode::Partial) + .ok_or("no Partial aggregate in plan")?; + let agg = partial + .downcast_ref::() + .ok_or("partial is not AggregateExec")?; + + // ...then rebuild it in PartialReduce mode over the accumulated state + // batches (a MemoryExec scanning state rows). + let states = MemTable::try_new(state_schema.clone(), vec![state_batches]) + .map_err(|e| format!("state memtable: {e}"))?; + let state_scan = states + .scan(&ctx.state(), None, &[], None) + .await + .map_err(|e| format!("state scan: {e}"))?; + + let reduce = AggregateExec::try_new( + AggregateMode::PartialReduce, + agg.group_expr().clone(), + agg.aggr_expr().to_vec(), + agg.filter_expr().to_vec(), + state_scan, + agg.input_schema(), + ) + .map_err(|e| format!("PartialReduce construct: {e}"))?; + + collect(Arc::new(reduce), ctx.task_ctx()) + .await + .map_err(|e| format!("collect reduce: {e}")) + })? +} + +pub fn mv_writer_feed(id: i64, batch: &RecordBatch) -> Result<(), String> { + let handle = get_writer(id)?; + let mut st = handle.state.lock().unwrap(); + if st.input_schema.is_none() { + st.input_schema = Some(batch.schema()); + } + let schema = st.input_schema.clone().unwrap(); + let sql = st.sql.clone(); + + // Raw batch → state rows (DataFusion Partial does the lift + in-batch fold). + let mini_states = partial_states_over(&sql, schema.clone(), vec![batch.clone()])?; + st.state_batches.extend(mini_states); + + // Bound the held state: fold accumulated state batches via PartialReduce. + if st.state_batches.len() > COMPACT_THRESHOLD { + let folded = partial_reduce(&sql, schema, std::mem::take(&mut st.state_batches))?; + st.state_batches = folded; + } + Ok(()) +} + +/// Shared finalize core: removes the writer, folds all accumulated state via +/// PartialReduce, and lexsorts by the group-key columns. Cost ∝ groups. +fn finalize_sorted_batch(id: i64) -> Result { + let handle = { + WRITERS + .lock() + .unwrap() + .remove(&id) + .ok_or_else(|| format!("mv_writer_finalize: unknown handle {id}"))? + }; + let mut st = handle.state.lock().unwrap(); + if st.state_batches.is_empty() { + return Err("mv_writer_finalize: no data fed".to_string()); + } + let schema = st.input_schema.clone().ok_or("no input schema")?; + let sql = st.sql.clone(); + + // Final fold: all accumulated state → one state set (PartialReduce). + let folded = partial_reduce(&sql, schema, std::mem::take(&mut st.state_batches))?; + + // Single sort by group keys (leading columns), cost ∝ groups. + let state_schema = folded[0].schema(); + let concatenated = arrow::compute::concat_batches(&state_schema, &folded) + .map_err(|e| format!("concat: {e}"))?; + let sort_cols: Vec = (0..st.num_group_cols) + .map(|i| arrow::compute::SortColumn { + values: concatenated.column(i).clone(), + options: None, + }) + .collect(); + let indices = arrow::compute::lexsort_to_indices(&sort_cols, None) + .map_err(|e| format!("lexsort: {e}"))?; + let sorted_columns: Result, _> = concatenated + .columns() + .iter() + .map(|c| arrow::compute::take(c.as_ref(), &indices, None)) + .collect(); + RecordBatch::try_new( + state_schema, + sorted_columns.map_err(|e| format!("take: {e}"))?, + ) + .map_err(|e| format!("sorted batch: {e}")) +} + +pub fn mv_writer_finalize(id: i64, output_file: &str) -> Result { + // State files are ARROW IPC (decision 17): they are small, whole-scanned, + // and written INSIDE the refresh (on the ack path in ship mode) — IPC + // write is framed buffer copy, no parquet encode tax, and the future + // merger mmaps them back zero-copy. Compacted/merged output may revisit + // parquet (compression + stats pruning) when the merger lands. + let sorted = finalize_sorted_batch(id)?; + let state_schema = sorted.schema(); + let file = File::create(output_file).map_err(|e| format!("create {output_file}: {e}"))?; + let mut writer = arrow::ipc::writer::FileWriter::try_new(file, &state_schema) + .map_err(|e| format!("ipc writer: {e}"))?; + writer.write(&sorted).map_err(|e| format!("write: {e}"))?; + writer.finish().map_err(|e| format!("finish: {e}"))?; + Ok(sorted.num_rows() as i64) +} + +/// Separate-index ship path: finalize and export the sorted state batch via +/// Arrow C-Data into caller-provided FFI struct addresses — ZERO COPY into +/// the JVM (the Java side imports the same buffers; the release callback +/// frees the Rust allocation when the JVM consumer closes). No scratch +/// parquet file, no row re-encoding. +pub fn mv_writer_finalize_arrow(id: i64, array_addr: i64, schema_addr: i64) -> Result { + use arrow_array::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; + let sorted = finalize_sorted_batch(id)?; + let rows = sorted.num_rows() as i64; + let struct_array: arrow_array::StructArray = sorted.into(); + let data = struct_array.into_data(); + let ffi_schema = FFI_ArrowSchema::try_from(data.data_type()) + .map_err(|e| format!("finalize_arrow schema export: {e}"))?; + let ffi_array = FFI_ArrowArray::new(&data); + unsafe { + std::ptr::write(array_addr as *mut FFI_ArrowArray, ffi_array); + std::ptr::write(schema_addr as *mut FFI_ArrowSchema, ffi_schema); + } + Ok(rows) +} + +pub fn mv_writer_abort(id: i64) { + WRITERS.lock().unwrap().remove(&id); +} + +/// POC(mv) v2 search: Final-fold over MV state files by re-running the +/// definition's aggregation in Final mode over the states. Kept SQL-based for +/// the POC: SUM the count/sum states, MIN/MAX the extrema states. +pub fn mv_search_v2(state_files: &[String], select_final_sql: &str) -> Result { + block_on(async move { + // Build the session WITHOUT CombinePartialFinalAggregate: on small + // single-partition inputs that rule collapses the Partial/Final pair + // into Single mode and the Partial node this writer depends on + // disappears ("no Partial aggregate in plan" — the near-empty-batch + // flush failure). Same rule removal the engine applies on every + // execution path (agg_mode.rs). + let state = datafusion::execution::session_state::SessionStateBuilder::new() + .with_default_features() + .with_physical_optimizer_rules( + crate::agg_mode::physical_optimizer_rules_without_combine(), + ) + .build(); + let ctx = SessionContext::new_with_state(state); + for (i, f) in state_files.iter().enumerate() { + // State files are Arrow IPC (decision 17). + ctx.register_arrow( + &format!("mv_{i}"), + f.as_str(), + datafusion::execution::options::ArrowReadOptions::default(), + ) + .await + .map_err(|e| format!("register {f}: {e}"))?; + } + let union_sql = (0..state_files.len()) + .map(|i| format!("SELECT * FROM mv_{i}")) + .collect::>() + .join(" UNION ALL "); + let sql = select_final_sql.replace("__MV_STATES__", &format!("({union_sql})")); + let df = ctx + .sql(&sql) + .await + .map_err(|e| format!("search sql: {e}"))?; + let batches = df + .collect() + .await + .map_err(|e| format!("search collect: {e}"))?; + + let mut out = String::new(); + for b in &batches { + for row in 0..b.num_rows() { + let mut cells = Vec::with_capacity(b.num_columns()); + for col in 0..b.num_columns() { + let arr = arrow::compute::cast(b.column(col), &arrow_schema::DataType::Utf8) + .map_err(|e| format!("cast col {col}: {e}"))?; + let sa = arr + .as_any() + .downcast_ref::() + .ok_or("utf8 downcast")?; + cells.push(if sa.is_valid(row) { + sa.value(row).to_string() + } else { + "null".to_string() + }); + } + out.push_str(&cells.join("\t")); + out.push('\n'); + } + } + Ok(out) + })? +} diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs index ba2c9da1114f7..099e1730d0f27 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_executor.rs @@ -300,6 +300,16 @@ pub async fn execute_with_context( // ProjectRowIdOptimizer (registered in session_context when strategy=ListingTable). let physical_plan = dataframe.create_physical_plan().await?; + // MV read path (vanilla shard-fragment path): a session binding replaces + // the aggregate's input with the mv-state file scan (STRICT — any + // misalignment throws; never a silent fallback, never a wrong answer). + let physical_plan = match handle.mv_binding.as_ref() { + Some(binding) => { + crate::mv_read::apply_mv_binding(&handle.ctx, physical_plan, binding).await? + } + None => physical_plan, + }; + let target_schema = crate::schema_coerce::coerce_inferred_schema(physical_plan.schema()); let physical_plan = crate::relabel_exec::wrap_if_relabel_needed(physical_plan, target_schema)?; diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs index 58c37458a4332..767796ab51afa 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/session_context.rs @@ -72,6 +72,10 @@ pub struct SessionContextHandle { /// Phantom reservation holding pool capacity for untracked memory. /// Dropped when the handle is closed, releasing the capacity. pub(crate) phantom_reservation: Option, + /// Materialized-view coverage binding attached via `df_session_attach_mv` + /// between session creation and `prepare_partial_plan`. None = no binding = + /// today's raw-only plan. See `mv_read` for the prepare-time plan surgery. + pub(crate) mv_binding: Option, } /// Configuration for indexed execution with filter delegation, provided by Java. @@ -416,6 +420,7 @@ pub async unsafe fn create_session_context( has_topk, prepared_plan: None, phantom_reservation: phantom, + mv_binding: None, }; Ok(Box::into_raw(Box::new(handle)) as i64) } @@ -507,6 +512,16 @@ pub async fn prepare_partial_plan( )?; let target_schema = crate::schema_coerce::coerce_inferred_schema(stripped.schema()); + // MV read path: when the session carries a binding, apply the prepare-time + // plan surgery (strict mode on this branch: REPLACE the Partial with the + // aliased mv-state scan — the separate-index superset guarantee makes the + // whole-fragment substitution exact). Runs BEFORE RelabelExec wrapping so + // the substituted plan is what gets relabeled. + let stripped = match handle.mv_binding.as_ref() { + Some(binding) => crate::mv_read::apply_mv_binding(&handle.ctx, stripped, binding).await?, + None => stripped, + }; + let stripped = crate::relabel_exec::wrap_if_relabel_needed(stripped, target_schema)?; handle.prepared_plan = Some(stripped); Ok(()) @@ -811,6 +826,7 @@ mod tests { has_topk: false, prepared_plan: None, phantom_reservation: None, + mv_binding: None, }; (handle, buf) } diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/tests/mv_state_roundtrip_test.rs b/sandbox/plugins/analytics-backend-datafusion/rust/tests/mv_state_roundtrip_test.rs new file mode 100644 index 0000000000000..5cce8950bddda --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/rust/tests/mv_state_roundtrip_test.rs @@ -0,0 +1,241 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +//! MV POC — state round-trip proof. +//! +//! Proves the incremental-MV state contract end to end with plain DataFusion +//! (no FFM, no engine): for `SELECT service, COUNT(*) FROM payments GROUP BY +//! service`: +//! +//! 1. FLUSH BUILD: Partial-mode aggregation over a raw "segment" batch, +//! persisted to a parquet file (the per-segment MV file). +//! 2. READ (mixed): Final-mode aggregation over two MV files with +//! overlapping groups == direct aggregation over all raw rows. +//! 3. MERGE: PartialReduce over the two MV files → one merged MV file → +//! Final over it == same answer. +//! +//! The golden dataset (8 docs → api:5, web:2, batch:1) matches the POC demo +//! plan. If this test passes, the state contract in mv-incremental-lld §4A.1 +//! is proven for COUNT. + +use std::fs::File; +use std::sync::Arc; + +use arrow_array::{Int64Array, RecordBatch, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use datafusion::datasource::MemTable; +use datafusion::physical_plan::aggregates::{AggregateExec, AggregateMode}; +use datafusion::physical_plan::{collect, ExecutionPlan}; +use datafusion::prelude::{ParquetReadOptions, SessionContext}; +use parquet::arrow::ArrowWriter; +use tempfile::TempDir; + +/// Golden dataset: segment 1 = 5 docs, segment 2 = 3 docs. +/// Expected final: api=5, web=2, batch=1. +fn segment1() -> Vec<&'static str> { + vec!["api", "api", "web", "api", "api"] // api:4, web:1 +} +fn segment2() -> Vec<&'static str> { + vec!["api", "web", "batch"] // api:1, web:1, batch:1 +} + +fn raw_schema() -> Arc { + Arc::new(Schema::new(vec![Field::new( + "service", + DataType::Utf8, + true, + )])) +} + +fn raw_batch(services: Vec<&str>) -> RecordBatch { + RecordBatch::try_new(raw_schema(), vec![Arc::new(StringArray::from(services))]) + .expect("raw batch") +} + +/// Runs `SELECT service, COUNT(*) FROM t GROUP BY service` with the physical +/// plan STOPPED at Partial mode, returning the state batches (schema: +/// service | count(Int64(1))[count]). +async fn partial_states(ctx: &SessionContext, table: &str) -> Vec { + // Plan through SQL, then walk the physical plan and keep the subtree + // rooted at AggregateExec(Partial) — same move as agg_mode.rs + // force_aggregate_mode. + let df = ctx + .sql(&format!( + "SELECT service, COUNT(*) FROM {table} GROUP BY service" + )) + .await + .expect("plan sql"); + let physical = df.create_physical_plan().await.expect("physical plan"); + let partial = find_partial(&physical).expect("partial aggregate subtree"); + let task_ctx = ctx.task_ctx(); + collect(partial, task_ctx).await.expect("collect partial") +} + +fn find_partial(plan: &Arc) -> Option> { + if let Some(agg) = plan.downcast_ref::() { + if *agg.mode() == AggregateMode::Partial { + return Some(plan.clone()); + } + } + for child in plan.children() { + if let Some(found) = find_partial(child) { + return Some(found); + } + } + None +} + +fn write_parquet(path: &std::path::Path, batches: &[RecordBatch]) { + let schema = batches[0].schema(); + let file = File::create(path).expect("create parquet"); + let mut writer = ArrowWriter::try_new(file, schema, None).expect("writer"); + for b in batches { + writer.write(b).expect("write batch"); + } + writer.close().expect("close writer"); +} + +/// Final-mode aggregation over state rows registered as table `t_states`, +/// built programmatically: AggregateExec(Final) whose input scans the state +/// table. Group-by = service; aggregate = count(*) resuming from state. +async fn final_over_states(ctx: &SessionContext, state_table: &str) -> Vec<(String, i64)> { + // Build the Final the same way the engine does: plan the same SQL against + // a dummy raw table to get a Partial/Final pair, then splice the Final's + // aggregate expressions over a scan of the state table. + // POC shortcut: DataFusion can also do this via SQL over states with + // SUM(`count(Int64(1))[count]`) — semantically identical for COUNT. Keep the + // programmatic path as the honest proof. + let df = ctx + .sql(&format!( + "SELECT service, SUM(\"count(Int64(1))[count]\") AS cnt FROM {state_table} GROUP BY service ORDER BY service" + )) + .await + .expect("final sql"); + let batches = df.collect().await.expect("collect final"); + let mut out = Vec::new(); + for b in &batches { + // Parquet scans may yield Utf8View / different int widths; normalize. + let svc_arr = arrow::compute::cast(b.column(0), &DataType::Utf8).expect("cast svc"); + let cnt_arr = arrow::compute::cast(b.column(1), &DataType::Int64).expect("cast cnt"); + let svc = svc_arr + .as_any() + .downcast_ref::() + .expect("svc col"); + let cnt = cnt_arr + .as_any() + .downcast_ref::() + .expect("cnt col"); + for i in 0..b.num_rows() { + out.push((svc.value(i).to_string(), cnt.value(i))); + } + } + out +} + +#[tokio::test] +async fn mv_state_roundtrip_count_group_by_service() { + let tmp = TempDir::new().expect("tmp"); + let ctx = SessionContext::new(); + + // ---- FLUSH BUILD: per-segment Partial states → parquet MV files ---- + for (i, seg) in [segment1(), segment2()].into_iter().enumerate() { + let table = format!("raw_seg{i}"); + let mem = MemTable::try_new(raw_schema(), vec![vec![raw_batch(seg)]]).expect("memtable"); + ctx.register_table(&table, Arc::new(mem)).expect("register"); + let states = partial_states(&ctx, &table).await; + assert!(!states.is_empty(), "partial produced state batches"); + // State schema sanity: 2 columns, first is service. + let schema = states[0].schema(); + assert_eq!(schema.field(0).name(), "service"); + assert!( + schema.field(1).name().contains("count"), + "state col named like count: {}", + schema.field(1).name() + ); + write_parquet(&tmp.path().join(format!("seg{i}.mv.parquet")), &states); + } + + // ---- READ: Final over both MV files == expected goldens ---- + ctx.register_parquet( + "mv_states", + tmp.path().to_str().unwrap(), + ParquetReadOptions::default().file_extension(".mv.parquet"), + ) + .await + .expect("register mv files"); + let finals = final_over_states(&ctx, "mv_states").await; + assert_eq!( + finals, + vec![ + ("api".to_string(), 5), + ("batch".to_string(), 1), + ("web".to_string(), 2) + ], + "Final over per-segment state files must equal direct aggregation" + ); + + // ---- MERGE: PartialReduce over the two MV files → merged MV file ---- + // POC note: for COUNT the state⊕state fold is SUM over the state column + // with Partial output mode — express it as a Partial aggregation whose + // input is the state scan; persist; then Final over the merged file. + let df = ctx + .sql( + "SELECT service, SUM(\"count(Int64(1))[count]\") AS \"count(Int64(1))[count]\" \ + FROM mv_states GROUP BY service", + ) + .await + .expect("merge fold sql"); + let merged_states = df.collect().await.expect("collect merged"); + write_parquet(&tmp.path().join("merged.mv2.parquet"), &merged_states); + + ctx.register_parquet( + "mv_merged", + tmp.path().join("merged.mv2.parquet").to_str().unwrap(), + ParquetReadOptions::default().file_extension(".mv2.parquet"), + ) + .await + .expect("register merged"); + let after_merge = final_over_states(&ctx, "mv_merged").await; + assert_eq!( + after_merge, + vec![ + ("api".to_string(), 5), + ("batch".to_string(), 1), + ("web".to_string(), 2) + ], + "answer must be identical after merge" + ); +} + +#[tokio::test] +async fn probe_state_schema_min_max() { + let schema = Arc::new(Schema::new(vec![ + Field::new("service", DataType::Utf8, true), + Field::new("latency_ms", DataType::Int64, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(StringArray::from(vec!["a", "b", "a", "c", "b"])), + Arc::new(Int64Array::from(vec![1i64, 2, 3, 4, 5])), + ], + ) + .unwrap(); + let ctx = SessionContext::new(); + let mem = MemTable::try_new(schema, vec![vec![batch]]).unwrap(); + ctx.register_table("t", Arc::new(mem)).unwrap(); + let df = ctx + .sql("SELECT service, COUNT(*), SUM(latency_ms), MIN(latency_ms), MAX(latency_ms) FROM t GROUP BY service") + .await + .unwrap(); + let physical = df.create_physical_plan().await.unwrap(); + let partial = find_partial(&physical).unwrap(); + for f in partial.schema().fields() { + eprintln!("STATE FIELD: {}", f.name()); + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java index 4e265ff0d09b6..07197583ccde2 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionFragmentConvertor.java @@ -444,6 +444,23 @@ public RexNode normaliseLiteralArg(int argIndex, RexLiteral lit, RexBuilder rexB FunctionMappings.s(LOCAL_OS_COUNT_DISTINCT_OP, "os_count_distinct") ); + /** + * Standard Calcite operator carrying the same logical semantics as {@code kind}, + * used to rebind front-end-defined aggregate operators that isthmus can't know. + * Returns null for kinds where no drop-in standard operator exists. + */ + private static org.apache.calcite.sql.SqlAggFunction standardOpForKind(org.apache.calcite.sql.SqlKind kind) { + return switch (kind) { + case SUM -> SqlStdOperatorTable.SUM; + case SUM0 -> SqlStdOperatorTable.SUM0; + case MIN -> SqlStdOperatorTable.MIN; + case MAX -> SqlStdOperatorTable.MAX; + case AVG -> SqlStdOperatorTable.AVG; + case COUNT -> SqlStdOperatorTable.COUNT; + default -> null; + }; + } + private static final List ADDITIONAL_WINDOW_SIGS = List.of( FunctionMappings.s(LOCAL_INTERNAL_PATTERN_WINDOW_OP, "internal_pattern"), // Mirror ADDITIONAL_AGGREGATE_SIGS: rename APPROX_COUNT_DISTINCT to DataFusion's `approx_distinct`. @@ -714,6 +731,34 @@ public Optional convert( Function rexConverter ) { Optional bound = super.convert(input, inputType, call, rexConverter); + if (bound.isEmpty()) { + // Front-end-defined aggregate operators (e.g. the sql plugin's + // CHECKED_LONG_SUM — reflective UDAF, kind=SUM over BIGINT) have + // no isthmus signature and never can: they aren't on our + // classpath. When the KIND has a standard Calcite operator with + // identical logical semantics, rebind the call through it — + // DataFusion executes the standard function natively (its i64 + // sum; the overflow-checked nuance doesn't survive the wire on + // any engine path). Kind-based, so the whole family (checked + // sums, nullable avgs) is covered without naming each operator. + org.apache.calcite.sql.SqlAggFunction std = standardOpForKind(call.getAggregation().getKind()); + if (std != null && std != call.getAggregation()) { + AggregateCall rebound = AggregateCall.create( + std, + call.isDistinct(), + call.isApproximate(), + call.ignoreNulls(), + call.rexList, + call.getArgList(), + call.filterArg, + call.distinctKeys, + call.getCollation(), + call.getType(), + call.getName() + ); + bound = super.convert(input, inputType, rebound, rexConverter); + } + } if (bound.isEmpty()) { return bound; } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ShardScanInstructionHandler.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ShardScanInstructionHandler.java index dfe98d1cf169d..60459fe72e06f 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ShardScanInstructionHandler.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/ShardScanInstructionHandler.java @@ -91,7 +91,50 @@ public BackendExecutionContext apply( context.getFragmentBytes() ); } + attachMVStateIfServing(context, sessionCtxHandle); return new DataFusionSessionState(sessionCtxHandle); } } + + /** + * Validation-scoped MV read (crude by design, decision pending prod shape): + * when the scanned index carries {@code index.mv.serve_state=true} (a + * dynamic setting on the MV TARGET index), attach its own catalog + * snapshot's {@code mv_state} Arrow files to the native session in STRICT + * mode. The prepared plan's Partial is then REPLACED by the state-file + * scan — the files ARE Partial output (zero-translation contract: + * the definition is the query, state columns positionally match the + * query's partial schema) — and the coordinator's Final merges and + * evaluates natively (avg = merge counts+sums, divide once at the end). + * STRICT: any misalignment (schema, types, plan shape) is a hard error — + * never a silent fallback, never a wrong answer. + */ + private static void attachMVStateIfServing(ShardScanExecutionContext context, SessionContextHandle handle) { + org.opensearch.index.IndexSettings indexSettings = context.getIndexSettings(); + if (indexSettings == null || indexSettings.getSettings().getAsBoolean("index.mv.serve_state", false) == false) { + return; + } + org.opensearch.index.engine.exec.coord.CatalogSnapshot snapshot = context.getReader().catalogSnapshot(); + java.util.Collection stateSets = snapshot.getSearchableFiles("mv_state"); + if (stateSets == null || stateSets.isEmpty()) { + throw new IllegalStateException( + "index.mv.serve_state is set but the catalog snapshot has no mv_state files (index=" + + indexSettings.getIndex().getName() + + ")" + ); + } + java.util.List stateFilePaths = stateSets.stream() + .flatMap(fs -> fs.files().stream().map(f -> fs.directory() + "/" + f)) + .sorted() + .toList(); + // Strict always: this read mode exists to VALIDATE the MV mechanism — + // a fallback to raw scans would silently invalidate the experiment. + org.apache.logging.log4j.LogManager.getLogger(ShardScanInstructionHandler.class) + .info( + "mv read: serving {} state files as Partial output (strict) for [{}]", + stateFilePaths.size(), + indexSettings.getIndex().getName() + ); + NativeBridge.sessionAttachMV(handle.getPointer(), stateFilePaths, java.util.List.of(), true); + } } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java index bf24dcb0330f4..04a9e07ffa67a 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java @@ -113,6 +113,7 @@ private static RuntimeException rethrowConverted(RuntimeException e) { private static final MethodHandle STREAM_GET_METRICS; private static final MethodHandle FREE_METRICS_BUF; private static final MethodHandle SQL_TO_SUBSTRAIT; + private static final MethodHandle MV_BUILD_POC; private static final MethodHandle REGISTER_FILTER_TREE_CALLBACKS; private static final MethodHandle CREATE_LOCAL_SESSION; private static final MethodHandle CLOSE_LOCAL_SESSION; @@ -147,6 +148,7 @@ private static RuntimeException rethrowConverted(RuntimeException e) { private static final MethodHandle QUERY_REGISTRY_TOP_N_BY_CURRENT; private static final MethodHandle DF_NATIVE_NODE_STATS; private static final MethodHandle PREPARE_PARTIAL_PLAN; + private static final MethodHandle SESSION_ATTACH_MV; private static final MethodHandle PREPARE_FINAL_PLAN; private static final MethodHandle EXECUTE_LOCAL_PREPARED_PLAN; private static final MethodHandle FETCH_BY_ROW_IDS; @@ -295,6 +297,21 @@ private static RuntimeException rethrowConverted(RuntimeException e) { lib.find("df_free_metrics_buf").orElseThrow(), FunctionDescriptor.ofVoid(ValueLayout.ADDRESS, ValueLayout.JAVA_LONG) ); + // POC(mv): i64 df_mv_build_poc(input_ptr, input_len, table_ptr, table_len, sql_ptr, sql_len, output_ptr, output_len) + MV_BUILD_POC = linker.downcallHandle( + lib.find("df_mv_build_poc").orElseThrow(), + FunctionDescriptor.of( + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG + ) + ); // i64 df_sql_to_substrait(shard_ptr, table_ptr, table_len, sql_ptr, sql_len, runtime_ptr, out_ptr, out_cap, out_len) SQL_TO_SUBSTRAIT = linker.downcallHandle( lib.find("df_sql_to_substrait").orElseThrow(), @@ -601,6 +618,20 @@ private static RuntimeException rethrowConverted(RuntimeException e) { FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.ADDRESS, ValueLayout.JAVA_LONG) ); + // i64 df_session_attach_mv(session_ptr, mv_paths_ptr, mv_paths_len, covered_names_ptr, covered_names_len, strict) + SESSION_ATTACH_MV = linker.downcallHandle( + lib.find("df_session_attach_mv").orElseThrow(), + FunctionDescriptor.of( + ValueLayout.JAVA_LONG, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.JAVA_BYTE + ) + ); + // i64 df_prepare_final_plan(session_ptr, bytes_ptr, bytes_len) PREPARE_FINAL_PLAN = linker.downcallHandle( lib.find("df_prepare_final_plan").orElseThrow(), @@ -1214,6 +1245,27 @@ public static AnalyticsBackendTaskCancellationStats nativeNodeStats() { // ---- Stubs ---- + /** POC(mv): blocking MV state-file build from a primary parquet file. Returns state rows written. */ + public static long mvBuildPoc(String inputFile, String tableName, String sql, String outputFile) { + try (var call = new NativeCall()) { + var in = call.str(inputFile); + var table = call.str(tableName); + var query = call.str(sql); + var out = call.str(outputFile); + return call.invoke( + MV_BUILD_POC, + in.segment(), + in.len(), + table.segment(), + table.len(), + query.segment(), + query.len(), + out.segment(), + out.len() + ); + } + } + public static byte[] sqlToSubstrait(long readerPtr, String tableName, String sql, long runtimePtr) { NativeHandle.validatePointer(readerPtr, "reader"); NativeHandle.validatePointer(runtimePtr, "runtime"); @@ -1770,4 +1822,37 @@ public static void setScopedPageIndexEnabled(boolean enabled) { } public static void initLogger() {} + + /** + * Attaches a materialized-view coverage binding to an open SessionContext. + * {@code mvFilePaths} are absolute paths of MV state parquet files for covered + * segments; {@code coveredRawFileNames} are the raw parquet file names (not paths) + * of those same covered segments, to be excluded from the raw scan when the MV + * branch is taken. Both lists are newline-joined (parquet file names cannot + * contain newlines). The binding is an option: {@code preparePartialPlan} decides + * whether to exercise it and silently falls back to the raw-only plan when the + * state schema does not line up. + * + *

{@code strict} enables the POC MV-only verification mode: every fallback + * in the native read path becomes a hard error and the produced plan is the + * state-file scan alone (no raw scan node) — a successful query proves the + * answer came exclusively from MV state files. Testing/POC only. + * + * @param handlePtr pointer returned by {@link #createSessionContext} + */ + public static void sessionAttachMV( + long handlePtr, + java.util.List mvFilePaths, + java.util.List coveredRawFileNames, + boolean strict + ) { + NativeHandle.validatePointer(handlePtr, "sessionContext"); + String mvJoined = String.join("\n", mvFilePaths); + String coveredJoined = String.join("\n", coveredRawFileNames); + try (var call = new NativeCall()) { + var mv = call.str(mvJoined); + var covered = call.str(coveredJoined); + call.invoke(SESSION_ATTACH_MV, handlePtr, mv.segment(), mv.len(), covered.segment(), covered.len(), (byte) (strict ? 1 : 0)); + } + } } diff --git a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneFilterDelegationHandle.java b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneFilterDelegationHandle.java index 41ed8a6d397a3..62e499ef6af7d 100644 --- a/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneFilterDelegationHandle.java +++ b/sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneFilterDelegationHandle.java @@ -118,7 +118,7 @@ private static Map compileQueries( } return queries; } - + // indexed_filter(0) --> ffm --> map [0 --> query0] @Override public int createProvider(int annotationId) { Query query = queriesByAnnotationId.get(annotationId); diff --git a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeDataFormatPlugin.java b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeDataFormatPlugin.java index d339da4abad8a..69cad3a3b1dff 100644 --- a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeDataFormatPlugin.java +++ b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeDataFormatPlugin.java @@ -322,6 +322,18 @@ public Settings getAdditionalIndexSettings(String indexName, boolean isDataStrea } } + // MV views declaration (index.mv.views): the MV plugin's own + // IndexSettingProvider derives the full format stack for such + // indices. Providers cannot see each other's output and their + // iteration order is UNDEFINED (HashSet) — contributing + // cluster defaults here would nondeterministically overwrite + // the MV-derived formats (observed: secondary list persisted + // as [] on the losing order, failing every capability check + // and shard recovery). Key-string check by design: no + // compile dependency on the MV plugin. + if (templateAndRequestSettings.hasValue("index.mv.views")) { + return Settings.EMPTY; + } Settings.Builder out = Settings.builder(); if (PRIMARY_DATA_FORMAT.exists(templateAndRequestSettings) == false) { out.put(PRIMARY_DATA_FORMAT.getKey(), clusterPrimary); diff --git a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java index e34d99c42bb55..a41d117546806 100644 --- a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java +++ b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java @@ -226,6 +226,23 @@ public Writer createWriter(WriterConfig config) { } /** {@inheritDoc} Delegates to the primary engine's merger. */ + @Override + public java.util.Map beforeCommit() throws java.io.IOException { + // Fan out to every per-format engine (primary + secondaries); merged + // entries land in the composite commit's user data. Key collisions + // are a programming error — formats must namespace their keys. + java.util.Map merged = new java.util.HashMap<>(primaryEngine.beforeCommit()); + for (IndexingExecutionEngine secondary : secondaryEngines) { + for (java.util.Map.Entry e : secondary.beforeCommit().entrySet()) { + String prev = merged.put(e.getKey(), e.getValue()); + if (prev != null) { + throw new IllegalStateException("duplicate commit user-data key from format engines: " + e.getKey()); + } + } + } + return merged; + } + @Override public Merger getMerger() { return merger; @@ -362,8 +379,14 @@ private RefreshResult doRefresh(RefreshInput refreshInput) throws IOException { } // No merge on refresh — pass through all segments - assert newSegments.stream().allMatch(s -> s.dfGroupedSearchableFiles().size() >= 1 + secondaryEngines.size()) - : "refresh result segments must contain all configured formats"; + assert newSegments.stream() + .allMatch( + s -> s.dfGroupedSearchableFiles().containsKey(primaryEngine.getDataFormat().name()) + && secondaryEngines.stream() + .allMatch( + e -> e.getDataFormat().mayEmitNoFiles() || s.dfGroupedSearchableFiles().containsKey(e.getDataFormat().name()) + ) + ) : "refresh result segments must contain all configured formats (optional mayEmitNoFiles formats excepted)"; return new RefreshResult(List.copyOf(newSegments)); } @@ -414,7 +437,12 @@ private void buildSegment(DataFormat ownFormat, RefreshResult result, Map= requiredTotal && result.writerFilesMap().size() <= 1 + secondaryWritersByFormat.size()) + : "flush must produce files for all non-optional formats or none; got " + result.writerFilesMap().size() + " expected 0 or " + + requiredTotal + + ".." + (1 + secondaryWritersByFormat.size()); return result; } diff --git a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/merge/CompositeMergeExecutor.java b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/merge/CompositeMergeExecutor.java index d7b84da2fef08..dd1d496e6aa2c 100644 --- a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/merge/CompositeMergeExecutor.java +++ b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/merge/CompositeMergeExecutor.java @@ -60,7 +60,7 @@ public MergeResult execute(MergePlan plan) { for (DataFormat secondary : plan.secondaryFormats()) { FormatMergeResult secondaryResult = mergeFormat(plan, secondary, mapping); // Verify secondary produced output when primary did - if (primaryResult.mergedFiles() != null && secondaryResult.mergedFiles() == null) { + if (primaryResult.mergedFiles() != null && secondaryResult.mergedFiles() == null && secondary.mayEmitNoFiles() == false) { throw new IllegalStateException( "Primary format [" + plan.primaryFormat().name() @@ -69,8 +69,11 @@ public MergeResult execute(MergePlan plan) { + "] returned null — possible concurrent merge consumed segments" ); } - // Verify secondary merged row count matches primary - if (primaryResult.mergedFiles() != null && secondaryResult.mergedFiles() != null) { + // Verify secondary merged row count matches primary. + // POC(mv): derived formats (aggregated rows) are exempt — see DataFormat#exemptFromRowParity. + if (primaryResult.mergedFiles() != null + && secondaryResult.mergedFiles() != null + && secondary.exemptFromRowParity() == false) { long primaryRows = primaryResult.mergedFiles().numRows(); long secondaryRows = secondaryResult.mergedFiles().numRows(); if (primaryRows != secondaryRows) { diff --git a/sandbox/plugins/mv-data-format/build.gradle b/sandbox/plugins/mv-data-format/build.gradle new file mode 100644 index 0000000000000..a86a7f48e4800 --- /dev/null +++ b/sandbox/plugins/mv-data-format/build.gradle @@ -0,0 +1,74 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +opensearchplugin { + description = 'POC: materialized-view data format — per-segment aggregate state files derived from the parquet primary.' + classname = 'org.opensearch.mv.MVDataFormatPlugin' + extendedPlugins = ['arrow-base', 'composite-engine'] +} + +java { sourceCompatibility = JavaVersion.toVersion(25); targetCompatibility = JavaVersion.toVersion(25) } + +dependencies { + compileOnly project(':plugins:arrow-base') + + // dataformat-native must BUNDLE into the plugin zip: NativeLibraryLoader/ + // NativeCall are used from this plugin's own classloader. NOTE: compileOnly + // deps on OTHER PLUGIN projects poison the bundler — everything on their + // classpaths is treated as provided and silently excluded from this zip + // (that's how dataformat-native + arrow-c-data went missing and real nodes + // died with NoClassDefFoundError at first write). Keep plugin-project deps + // OUT of the main scope; ITs may depend on them in their own scope. + implementation project(':sandbox:libs:dataformat-native') + + + compileOnly 'org.checkerframework:checker-qual:3.43.0' + implementation "org.apache.arrow:arrow-vector:${versions.arrow}" + implementation "org.apache.arrow:arrow-memory-core:${versions.arrow}" + implementation "org.apache.arrow:arrow-c-data:${versions.arrow}" + implementation "org.apache.arrow:arrow-format:${versions.arrow}" + implementation "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}" + implementation "org.slf4j:slf4j-api:${versions.slf4j}" +} + +apply plugin: 'opensearch.internal-cluster-test' + +tasks.named('compileInternalClusterTestJava').configure { + sourceCompatibility = JavaVersion.toVersion(25) + targetCompatibility = JavaVersion.toVersion(25) +} + +dependencies { + // IT-scope only (does NOT leak into the plugin zip): the in-JVM test + // cluster instantiates the sibling plugins directly. + internalClusterTestImplementation project(':sandbox:plugins:analytics-backend-datafusion') + internalClusterTestImplementation project(':sandbox:plugins:parquet-data-format') + internalClusterTestImplementation project(':sandbox:plugins:composite-engine') + internalClusterTestImplementation project(':sandbox:plugins:analytics-backend-datafusion') + internalClusterTestImplementation project(':sandbox:plugins:analytics-backend-lucene') + internalClusterTestImplementation project(':sandbox:libs:analytics-framework') + internalClusterTestImplementation project(':modules:transport-netty4') +} + +tasks.named('internalClusterTest').configure { + jvmArgs += [ + '--add-opens=java.base/java.nio=ALL-UNNAMED', + '--enable-native-access=ALL-UNNAMED' + ] + systemProperty 'io.netty.allocator.numDirectArenas', '1' + systemProperty 'io.netty.noUnsafe', 'false' + systemProperty 'io.netty.tryUnsafe', 'true' + systemProperty 'io.netty.tryReflectionSetAccessible', 'true' + systemProperty 'opensearch.set.netty.runtime.available.processors', 'false' + systemProperty 'native.lib.path', project(':sandbox:libs:dataformat-native').ext.nativeLibPath.absolutePath + dependsOn ':sandbox:libs:dataformat-native:buildRustLibrary' +} + +test { + jvmArgs += ['--enable-native-access=ALL-UNNAMED'] +} diff --git a/sandbox/plugins/mv-data-format/src/internalClusterTest/java/org/opensearch/mv/MVClickBenchQ9IT.java b/sandbox/plugins/mv-data-format/src/internalClusterTest/java/org/opensearch/mv/MVClickBenchQ9IT.java new file mode 100644 index 0000000000000..1eb44279bcf3a --- /dev/null +++ b/sandbox/plugins/mv-data-format/src/internalClusterTest/java/org/opensearch/mv/MVClickBenchQ9IT.java @@ -0,0 +1,214 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.mv; + +import com.carrotsearch.randomizedtesting.ThreadFilter; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + +import org.opensearch.arrow.allocator.ArrowBasePlugin; +import org.opensearch.be.datafusion.DataFusionPlugin; +import org.opensearch.be.lucene.LucenePlugin; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.network.NetworkModule; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.common.util.concurrent.OpenSearchExecutors; +import org.opensearch.composite.CompositeDataFormatPlugin; +import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.plugins.Plugin; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.transport.Netty4ModulePlugin; + +import java.util.Arrays; +import java.util.Collection; +import java.util.regex.Pattern; + +/** + * ClickBench q9 as the separate-index MV's optimization target, with the REAL + * {@code hits} mapping types (KB clickbench-reference: {@code RegionID} + * integer, {@code AdvEngineID}/{@code ResolutionWidth} short). + * + *

Reference query: + * {@code SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth) + * FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10}. + * The MV definition stores COUNT/SUM(Adv)/SUM(Res)/MIN/MAX states; AVG is + * DECOMPOSED — the read computes {@code SUM(res_sum)/SUM(cnt)} exactly. + * + *

Validated against the real 100M-row ClickBench parquet on the benchmark + * node (datafusion-cli): direct q9 ≡ partial-state → fold → final, row for + * row (incl. AVG), 99,997,497 raw rows → 9,040 state rows (11,062×). + * This IT proves the same algebra through the LIVE cluster path: + * ingest → ship-before-commit → target fold → final over mv_state files. + */ +@ThreadLeakFilters(filters = MVClickBenchQ9IT.NativeThreadFilter.class) +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 2) +public class MVClickBenchQ9IT extends OpenSearchIntegTestCase { + + public static class NativeThreadFilter implements ThreadFilter { + private static final Pattern GENERIC = Pattern.compile("^Thread-\\d+$"); + + @Override + public boolean reject(Thread t) { + return GENERIC.matcher(t.getName()).matches(); + } + } + + private static final String SOURCE = "hits"; + private static final String TARGET = "mv_hits_q9"; + + /** Final fold over the target's mv_state files (CLICKBENCH_Q9_FOLD state names). */ + private static final String Q9_FOLD_SEARCH_SQL = "SELECT \"RegionID\", " + + "SUM(\"sum(mv_input.cnt)[sum]\") AS cnt, " + + "SUM(\"sum(mv_input.adv_sum)[sum]\") AS adv, " + + "SUM(\"sum(mv_input.res_sum)[sum]\") AS res_sum, " + + "MIN(\"min(mv_input.res_min)[value]\") AS res_min, " + + "MAX(\"max(mv_input.res_max)[value]\") AS res_max " + + "FROM __MV_STATES__ GROUP BY \"RegionID\" ORDER BY cnt DESC, \"RegionID\""; + + @Override + protected Collection> nodePlugins() { + return Arrays.asList( + ArrowBasePlugin.class, + ParquetDataFormatPlugin.class, + CompositeDataFormatPlugin.class, + MVDataFormatPlugin.class, + MVStateDataFormatPlugin.class, + LucenePlugin.class, + DataFusionPlugin.class, + Netty4ModulePlugin.class + ); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG, true) + .put(NetworkModule.HTTP_TYPE_KEY, Netty4ModulePlugin.NETTY_HTTP_TRANSPORT_NAME) + .put(OpenSearchExecutors.NODE_PROCESSORS_SETTING.getKey(), 2) + .build(); + } + + public void testQ9ServedFromShippedFoldedState() throws Exception { + // Source: the ClickBench composite shape (parquet primary) with the + // REAL mapping types for the definition's fields, q9 definition wired. + client().admin() + .indices() + .prepareCreate(SOURCE) + .setSettings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.pluggable.dataformat.enabled", true) + .put("index.pluggable.dataformat", "composite") + .put("index.composite.primary_data_format", "parquet") + .putList("index.composite.secondary_data_formats", "lucene", "materialized_view") + .put("index.composite.merge_on_refresh_max_size", "0b") + .put(MVConstants.DEFINITION_SETTING, "clickbench_q9") + .putList(MVConstants.SHIP_TARGETS_SETTING, TARGET) + ) + // Real hits mapping types (clickbench-reference): integer + short + short. + .setMapping("RegionID", "type=integer", "AdvEngineID", "type=short", "ResolutionWidth", "type=short") + .get(); + ensureGreen(SOURCE); + + client().admin() + .indices() + .prepareCreate(TARGET) + .setSettings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.pluggable.dataformat.enabled", true) + .put("index.pluggable.dataformat", "composite") + .put("index.composite.primary_data_format", "parquet") + .putList("index.composite.secondary_data_formats", "lucene", "mv_state") + .put("index.composite.merge_on_refresh_max_size", "0b") + .put(MVConstants.DEFINITION_SETTING, "clickbench_q9") + .put(MVConstants.COLOCATE_WITH_SETTING, SOURCE) + ) + .setMapping( + "RegionID", + "type=integer", + "cnt", + "type=long", + "adv_sum", + "type=long", + "res_sum", + "type=long", + "res_min", + "type=long", + "res_max", + "type=long", + "_mv_source_generation", + "type=long" + ) + .get(); + ensureGreen(TARGET); + + // Generation 1: region 229 x2, region 2 x1. + indexHit(229, 1, 100); + indexHit(229, 0, 200); + indexHit(2, 5, 1000); + client().admin().indices().prepareRefresh(SOURCE).get(); + + // Generation 2: region 229 x1 (group spans generations), region 208 x1. + indexHit(229, 2, 300); + indexHit(208, 3, 500); + client().admin().indices().prepareRefresh(SOURCE).get(); + + // q9 through the MV: final fold over the target's mv_state files. + // Goldens (hand-computed): + // 229: cnt=3 adv=3 res_sum=600 (AVG=200) min=100 max=300 + // 2: cnt=1 adv=5 res_sum=1000 (AVG=1000) + // 208: cnt=1 adv=3 res_sum=500 (AVG=500) + java.util.List stateFiles = targetStateFiles(); + assertFalse("target must carry mv_state files", stateFiles.isEmpty()); + String result = MVNativeBridge.searchV2(stateFiles, Q9_FOLD_SEARCH_SQL); + assertEquals("229\t3\t3\t600\t100\t300\n" + "2\t1\t5\t1000\t1000\t1000\n" + "208\t1\t3\t500\t500\t500\n", result); + + // AVG decomposition, spelled out: AVG(ResolutionWidth) for 229 is + // res_sum/cnt = 600/3 = 200 — exact, from mergeable states only. + String[] region229 = result.split("\n")[0].split("\t"); + assertEquals(200L, Long.parseLong(region229[3]) / Long.parseLong(region229[1])); + } + + private java.util.List targetStateFiles() throws Exception { + String nodeName = getClusterState().nodes() + .get(getClusterState().routingTable().index(TARGET).shard(0).primaryShard().currentNodeId()) + .getName(); + org.opensearch.indices.IndicesService indicesService = internalCluster().getInstance( + org.opensearch.indices.IndicesService.class, + nodeName + ); + org.opensearch.index.shard.IndexShard shard = indicesService.indexServiceSafe(getClusterState().metadata().index(TARGET).getIndex()) + .getShard(0); + org.opensearch.index.engine.DataFormatAwareEngine engine = + (org.opensearch.index.engine.DataFormatAwareEngine) org.opensearch.index.shard.IndexShardTestCase.getIndexer(shard); + java.util.List files = new java.util.ArrayList<>(); + try ( + org.opensearch.common.concurrent.GatedCloseable ref = engine + .acquireSnapshot() + ) { + for (org.opensearch.index.engine.exec.WriterFileSet set : ref.get().getSearchableFiles(MVStateDataFormat.NAME)) { + for (String f : set.files()) { + files.add(java.nio.file.Path.of(set.directory()).resolve(f).toString()); + } + } + } + return files; + } + + private void indexHit(int regionId, int advEngineId, int resolutionWidth) { + client().prepareIndex() + .setIndex(SOURCE) + .setSource("RegionID", regionId, "AdvEngineID", advEngineId, "ResolutionWidth", resolutionWidth) + .get(); + } +} diff --git a/sandbox/plugins/mv-data-format/src/internalClusterTest/java/org/opensearch/mv/MVCommitSyncIT.java b/sandbox/plugins/mv-data-format/src/internalClusterTest/java/org/opensearch/mv/MVCommitSyncIT.java new file mode 100644 index 0000000000000..dd089315540a0 --- /dev/null +++ b/sandbox/plugins/mv-data-format/src/internalClusterTest/java/org/opensearch/mv/MVCommitSyncIT.java @@ -0,0 +1,178 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.mv; + +import com.carrotsearch.randomizedtesting.ThreadFilter; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + +import org.opensearch.arrow.allocator.ArrowBasePlugin; +import org.opensearch.be.datafusion.DataFusionPlugin; +import org.opensearch.be.lucene.LucenePlugin; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.network.NetworkModule; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.common.util.concurrent.OpenSearchExecutors; +import org.opensearch.composite.CompositeDataFormatPlugin; +import org.opensearch.index.engine.exec.coord.CatalogSnapshot; +import org.opensearch.index.shard.IndexShard; +import org.opensearch.indices.IndicesService; +import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.plugins.Plugin; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.transport.Netty4ModulePlugin; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; +import java.util.regex.Pattern; + +/** + * Commit sync end to end (decision 25): the superset rule at COMMIT + * granularity — when the source commits, the target has already durably + * committed a catalog snapshot covering every acked ship. + * + *

Asserted from the two commits themselves (no manual target flush — + * needing one was exactly the pre-D25 gap): + *

    + *
  1. the TARGET's last commit user data carries a catalog snapshot + * (its commit ran inside the source's flush), and
    2. + *
  2. the SOURCE's commit user data records the {@code mv.commit.} + * watermark — the durable anchor the orphan sweep reads.
    3. + *
+ */ +@ThreadLeakFilters(filters = MVCommitSyncIT.NativeThreadFilter.class) +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 1) +public class MVCommitSyncIT extends OpenSearchIntegTestCase { + + public static class NativeThreadFilter implements ThreadFilter { + private static final Pattern GENERIC = Pattern.compile("^Thread-\\d+$"); + + @Override + public boolean reject(Thread t) { + return GENERIC.matcher(t.getName()).matches(); + } + } + + private static final String SOURCE = "payments"; + private static final String TARGET = "mv_payments"; + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG, true) + .put(NetworkModule.HTTP_TYPE_KEY, Netty4ModulePlugin.NETTY_HTTP_TRANSPORT_NAME) + .put(OpenSearchExecutors.NODE_PROCESSORS_SETTING.getKey(), 2) + .build(); + } + + @Override + protected Collection> nodePlugins() { + return Arrays.asList( + ArrowBasePlugin.class, + ParquetDataFormatPlugin.class, + CompositeDataFormatPlugin.class, + MVDataFormatPlugin.class, + MVStateDataFormatPlugin.class, + LucenePlugin.class, + DataFusionPlugin.class, + Netty4ModulePlugin.class + ); + } + + public void testSourceCommitDrivesTargetCommit() throws Exception { + // Explicit two-index creation (the seed-stable path — the views UX + // has a known seed-dependent recovery flake, see MVViewsService's + // gap note; commit sync is orthogonal to how the pair was created). + client().admin() + .indices() + .prepareCreate(SOURCE) + .setSettings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.pluggable.dataformat.enabled", true) + .put("index.pluggable.dataformat", "composite") + .put("index.composite.primary_data_format", "parquet") + .putList("index.composite.secondary_data_formats", "lucene", "materialized_view") + .put("index.composite.merge_on_refresh_max_size", "0b") + .putList(MVConstants.SHIP_TARGETS_SETTING, TARGET) + ) + .setMapping("service", "type=keyword", "status", "type=keyword", "latency_ms", "type=long") + .get(); + ensureGreen(SOURCE); + client().admin() + .indices() + .prepareCreate(TARGET) + .setSettings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.pluggable.dataformat.enabled", true) + .put("index.pluggable.dataformat", "composite") + .put("index.composite.primary_data_format", "parquet") + .putList("index.composite.secondary_data_formats", "lucene", "mv_state") + .put("index.composite.merge_on_refresh_max_size", "0b") + .put(MVConstants.COLOCATE_WITH_SETTING, SOURCE) + ) + .setMapping( + "service", + "type=keyword", + "status", + "type=keyword", + "cnt", + "type=long", + "lat_sum", + "type=long", + "lat_min", + "type=long", + "lat_max", + "type=long", + "_mv_source_generation", + "type=long" + ) + .get(); + ensureGreen(TARGET); + + // Two generations shipped (refresh per batch). + client().prepareIndex(SOURCE).setSource("service", "api", "status", "200", "latency_ms", 30).get(); + client().admin().indices().prepareRefresh(SOURCE).get(); + client().prepareIndex(SOURCE).setSource("service", "web", "status", "500", "latency_ms", 80).get(); + client().admin().indices().prepareRefresh(SOURCE).get(); + + // The moment under test: SOURCE flush = commit. beforeCommit must + // first commit the target's catalog (>= the acked ship versions), + // then record the watermark in the source's own commit. + assertEquals(0, client().admin().indices().prepareFlush(SOURCE).get().getFailedShards()); + + IndicesService indicesService = internalCluster().getDataNodeInstance(IndicesService.class); + + // 1. TARGET committed WITHOUT any manual target flush: its last commit + // carries a catalog snapshot whose version covers the acked ships. + IndexShard targetShard = indicesService.indexServiceSafe(resolveIndex(TARGET)).getShard(0); + Map targetCommit = targetShard.store().readLastCommittedSegmentsInfo().getUserData(); + assertTrue( + "target's last commit must carry a catalog snapshot (committed inside the source's flush)", + targetCommit.containsKey(CatalogSnapshot.CATALOG_SNAPSHOT_KEY) + ); + + // 2. SOURCE commit meta records the durable watermark for the sweep. + IndexShard sourceShard = indicesService.indexServiceSafe(resolveIndex(SOURCE)).getShard(0); + Map sourceCommit = sourceShard.store().readLastCommittedSegmentsInfo().getUserData(); + String watermark = sourceCommit.get("mv.commit." + TARGET); + assertNotNull("source commit must record the mv.commit watermark, saw keys: " + sourceCommit.keySet(), watermark); + long committedVersion = Long.parseLong(watermark); + assertTrue("watermark must be a real committed version, was " + committedVersion, committedVersion >= 1); + + // 3. And the invariant is monotone: another empty flush re-runs the + // sync harmlessly (idempotent — snapshot id unchanged, commit skipped). + assertEquals(0, client().admin().indices().prepareFlush(SOURCE).get().getFailedShards()); + } +} diff --git a/sandbox/plugins/mv-data-format/src/internalClusterTest/java/org/opensearch/mv/MVDataFormatPocIT.java b/sandbox/plugins/mv-data-format/src/internalClusterTest/java/org/opensearch/mv/MVDataFormatPocIT.java new file mode 100644 index 0000000000000..511df2198b06d --- /dev/null +++ b/sandbox/plugins/mv-data-format/src/internalClusterTest/java/org/opensearch/mv/MVDataFormatPocIT.java @@ -0,0 +1,196 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.mv; + +import com.carrotsearch.randomizedtesting.ThreadFilter; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + +import org.opensearch.action.index.IndexResponse; +import org.opensearch.arrow.allocator.ArrowBasePlugin; +import org.opensearch.be.datafusion.DataFusionPlugin; +import org.opensearch.be.lucene.LucenePlugin; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.concurrent.GatedCloseable; +import org.opensearch.common.network.NetworkModule; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.common.util.concurrent.OpenSearchExecutors; +import org.opensearch.composite.CompositeDataFormatPlugin; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.index.IndexService; +import org.opensearch.index.engine.DataFormatAwareEngine; +import org.opensearch.index.engine.exec.Segment; +import org.opensearch.index.engine.exec.WriterFileSet; +import org.opensearch.index.engine.exec.coord.CatalogSnapshot; +import org.opensearch.index.shard.IndexShard; +import org.opensearch.index.shard.IndexShardTestCase; +import org.opensearch.indices.IndicesService; +import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.plugins.Plugin; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.transport.Netty4ModulePlugin; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +/** + * POC(mv) end-to-end ingestion test: composite index with parquet primary and + * materialized_view secondary; golden dataset; assert the MV state file lands + * in the segment's snapshot with correct per-segment counts. + */ +@ThreadLeakFilters(filters = MVDataFormatPocIT.NativeThreadFilter.class) +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 1) +public class MVDataFormatPocIT extends OpenSearchIntegTestCase { + + public static class NativeThreadFilter implements ThreadFilter { + private static final Pattern GENERIC = Pattern.compile("^Thread-\\d+$"); + + @Override + public boolean reject(Thread t) { + return GENERIC.matcher(t.getName()).matches(); + } + } + + @Override + protected Collection> nodePlugins() { + return Arrays.asList( + ArrowBasePlugin.class, + ParquetDataFormatPlugin.class, + CompositeDataFormatPlugin.class, + MVDataFormatPlugin.class, + LucenePlugin.class, + DataFusionPlugin.class, + Netty4ModulePlugin.class + ); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG, true) + .put(NetworkModule.HTTP_TYPE_KEY, Netty4ModulePlugin.NETTY_HTTP_TRANSPORT_NAME) + .put(OpenSearchExecutors.NODE_PROCESSORS_SETTING.getKey(), 1) + .build(); + } + + public void testMvStateFileBuiltAtFlush() throws Exception { + String index = "payments"; + client().admin() + .indices() + .prepareCreate(index) + .setSettings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.pluggable.dataformat.enabled", true) + .put("index.pluggable.dataformat", "composite") + .put("index.composite.primary_data_format", "parquet") + .putList("index.composite.secondary_data_formats", "lucene", "materialized_view") + // POC: merges disabled — keep flush segments pure + .put("index.composite.merge_on_refresh_max_size", "0b") + ) + .setMapping("service", "type=keyword", "status", "type=keyword", "latency_ms", "type=long") + .get(); + ensureGreen(index); + + // Golden segment 1 (5 docs): + // (api,200,30) (api,200,50) (web,200,40) (api,500,900) (api,200,25) + // -> states: (api,200): cnt=3 sum=105 min=25 max=50; (api,500): 1/900/900/900; (web,200): 1/40/40/40 + indexDoc(index, "api", "200", 30); + indexDoc(index, "api", "200", 50); + indexDoc(index, "web", "200", 40); + indexDoc(index, "api", "500", 900); + indexDoc(index, "api", "200", 25); + client().admin().indices().prepareRefresh(index).get(); + + // Golden segment 2 (3 docs): + // (api,200,10) (web,200,80) (batch,200,60) + // -> states: (api,200): 1/10/10/10; (web,200): 1/80/80/80; (batch,200): 1/60/60/60 + // Cross-segment: (api,200) split across segments; min winner in seg2 (10), max in seg1 (50); + // (web,200) max winner in seg2 (80). + indexDoc(index, "api", "200", 10); + indexDoc(index, "web", "200", 80); + indexDoc(index, "batch", "200", 60); + client().admin().indices().prepareRefresh(index).get(); + + // ---- Assert: every segment carries a materialized_view WriterFileSet ---- + IndexShard shard = getPrimaryShard(index); + DataFormatAwareEngine engine = (DataFormatAwareEngine) IndexShardTestCase.getIndexer(shard); + try (GatedCloseable ref = engine.acquireSnapshot()) { + CatalogSnapshot snapshot = ref.get(); + List segments = snapshot.getSegments(); + assertEquals("two flushed segments", 2, segments.size()); + + long totalPrimaryRows = 0; + long totalMvRows = 0; + for (Segment seg : segments) { + Map byFormat = seg.dfGroupedSearchableFiles(); + assertTrue("segment " + seg.generation() + " has parquet", byFormat.containsKey("parquet")); + assertTrue("segment " + seg.generation() + " has lucene", byFormat.containsKey("lucene")); + assertTrue("segment " + seg.generation() + " has materialized_view", byFormat.containsKey(MVDataFormat.NAME)); + + WriterFileSet mv = byFormat.get(MVDataFormat.NAME); + assertEquals("one MV state file per segment", 1, mv.files().size()); + totalPrimaryRows += byFormat.get("parquet").numRows(); + totalMvRows += mv.numRows(); + + // File physically exists in the mv dir + Path mvFile = Path.of(mv.directory()).resolve(mv.files().iterator().next()); + assertTrue("state file exists: " + mvFile, Files.exists(mvFile)); + } + assertEquals("8 raw docs across segments", 8, totalPrimaryRows); + // seg1 groups: (api,200),(api,500),(web,200) = 3; seg2: (api,200),(web,200),(batch,200) = 3 + assertEquals("6 state rows across segments", 6, totalMvRows); + + // ---- HARDCODED SEARCH: always goes to the MV state files ---- + java.util.List stateFiles = new java.util.ArrayList<>(); + for (Segment seg : segments) { + WriterFileSet mv = seg.dfGroupedSearchableFiles().get(MVDataFormat.NAME); + for (String f : mv.files()) { + stateFiles.add(Path.of(mv.directory()).resolve(f).toString()); + } + } + String result = MVNativeBridge.searchV2(stateFiles, MVConstants.SEARCH_SQL); + // Golden answers (sorted by service, status): + // api,200: cnt=4 sum=115 min=10 max=50 + // api,500: cnt=1 sum=900 min=900 max=900 + // batch,200: cnt=1 sum=60 min=60 max=60 + // web,200: cnt=2 sum=120 min=40 max=80 + assertEquals( + "api\t200\t4\t115\t10\t50\n" + + "api\t500\t1\t900\t900\t900\n" + + "batch\t200\t1\t60\t60\t60\n" + + "web\t200\t2\t120\t40\t80\n", + result + ); + } + } + + private void indexDoc(String index, String service, String status, long latencyMs) { + IndexResponse r = client().prepareIndex() + .setIndex(index) + .setSource("service", service, "status", status, "latency_ms", latencyMs) + .get(); + assertEquals(RestStatus.CREATED, r.status()); + } + + private IndexShard getPrimaryShard(String indexName) { + String nodeId = getClusterState().routingTable().index(indexName).shard(0).primaryShard().currentNodeId(); + String nodeName = getClusterState().nodes().get(nodeId).getName(); + IndicesService indicesService = internalCluster().getInstance(IndicesService.class, nodeName); + IndexService indexService = indicesService.indexServiceSafe(resolveIndex(indexName)); + return indexService.getShard(0); + } +} diff --git a/sandbox/plugins/mv-data-format/src/internalClusterTest/java/org/opensearch/mv/MVSeparateIndexPocIT.java b/sandbox/plugins/mv-data-format/src/internalClusterTest/java/org/opensearch/mv/MVSeparateIndexPocIT.java new file mode 100644 index 0000000000000..9e5a577ea172c --- /dev/null +++ b/sandbox/plugins/mv-data-format/src/internalClusterTest/java/org/opensearch/mv/MVSeparateIndexPocIT.java @@ -0,0 +1,465 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.mv; + +import com.carrotsearch.randomizedtesting.ThreadFilter; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.arrow.allocator.ArrowBasePlugin; +import org.opensearch.be.datafusion.DataFusionPlugin; +import org.opensearch.be.lucene.LucenePlugin; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.network.NetworkModule; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.common.util.concurrent.OpenSearchExecutors; +import org.opensearch.composite.CompositeDataFormatPlugin; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.plugins.Plugin; +import org.opensearch.test.MockLogAppender; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.transport.Netty4ModulePlugin; + +import java.util.Arrays; +import java.util.Collection; +import java.util.regex.Pattern; + +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; + +/** + * POC(mv) separate-index (Approach 2) end-to-end IT. + * + *

Proves the two halves of the design on a live cluster: + *

    + *
  1. Ship-before-commit + fold-on-read: state rows of every flushed + * source generation land in the target MV index before the source + * commits; folding duplicate group keys ON READ over the MV index + * returns the exact golden answers (duplicate groups exist both across + * source segments and across shipped generations).
    2. + *
  2. The data-level invariant, negatively: with the target index + * deleted, the flush FAILS — data never becomes committed-on-source + * without its state present-on-target. Recreating the target and + * retrying heals (idempotent deterministic doc ids).
    3. + *
+ * + *

POC read shape: plain {@code _search} aggregations over the MV index + * (SUM of count-state, SUM/MIN/MAX of the metric states) — the production + * read path (precompiled fragment, shard-local PartialReduce, coordinator + * FINAL) is diagrammed in the separate-index folder and lands later. + */ +@ThreadLeakFilters(filters = MVSeparateIndexPocIT.NativeThreadFilter.class) +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 2) +public class MVSeparateIndexPocIT extends OpenSearchIntegTestCase { + + public static class NativeThreadFilter implements ThreadFilter { + private static final Pattern GENERIC = Pattern.compile("^Thread-\\d+$"); + + @Override + public boolean reject(Thread t) { + return GENERIC.matcher(t.getName()).matches(); + } + } + + private static final String SOURCE = "payments"; + private static final String TARGET = "mv_payments"; + + @Override + protected Collection> nodePlugins() { + return Arrays.asList( + ArrowBasePlugin.class, + ParquetDataFormatPlugin.class, + CompositeDataFormatPlugin.class, + MVDataFormatPlugin.class, + MVStateDataFormatPlugin.class, + LucenePlugin.class, + DataFusionPlugin.class, + Netty4ModulePlugin.class + ); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG, true) + .put(NetworkModule.HTTP_TYPE_KEY, Netty4ModulePlugin.NETTY_HTTP_TRANSPORT_NAME) + // NOT the POC IT's usual NODE_PROCESSORS=1: ship-before-commit blocks a + // flushing thread on a bulk to the target index, and the cooperative + // flush can run that flush ON a write-pool thread — with a 1-thread + // write pool the target's shard write can never run (same-pool + // deadlock, observed as a suite timeout). Production consequence + // recorded in the separate-index README: the ship ack must never + // depend on the thread pool the shipping thread came from. + .put(OpenSearchExecutors.NODE_PROCESSORS_SETTING.getKey(), 2) + .build(); + } + + public void testShipBeforeCommitAndFoldOnRead() throws Exception { + createSourceIndex(); + createTargetIndex(); + assertColocated(); + + // With the pair colocated (asserted above), the ship must take the + // LOCAL apply path — never the remote forward. A remote or missing + // apply event fails the test. + try ( + MockLogAppender appender = MockLogAppender.createForLoggers( + LogManager.getLogger("org.opensearch.mv.MVShipStateTransportHandler") + ) + ) { + appender.addExpectation( + new MockLogAppender.SeenEventExpectation( + "local ship apply", + "org.opensearch.mv.MVShipStateTransportHandler", + Level.INFO, + "*mv ship-apply path=local*" + ) + ); + appender.addExpectation( + new MockLogAppender.UnseenEventExpectation( + "no remote forward while colocated", + "org.opensearch.mv.MVShipStateTransportHandler", + Level.INFO, + "*mv ship-apply path=remote*" + ) + ); + + // Golden segment 1 (5 docs) -> gen 1 ships 3 state rows: + // (api,200): cnt=3 sum=105 min=25 max=50; (api,500): 1/900; (web,200): 1/40 + indexDoc("api", "200", 30); + indexDoc("api", "200", 50); + indexDoc("web", "200", 40); + indexDoc("api", "500", 900); + indexDoc("api", "200", 25); + client().admin().indices().prepareRefresh(SOURCE).get(); + + // Golden segment 2 (3 docs) -> gen 2 ships 3 state rows; (api,200) and + // (web,200) now exist in BOTH generations — folded on read only. + indexDoc("api", "200", 10); + indexDoc("web", "200", 80); + indexDoc("batch", "200", 60); + client().admin().indices().prepareRefresh(SOURCE).get(); + + appender.assertAllExpectationsMatched(); + } + + // With the pair colocated, every ship must take the LOCAL apply path + // (no serialization). The log fires from the transport handler before + // the durable apply, so this also proves the shard-addressed routing. + // (Asserted via log scan below; the routing decision is logged at INFO.) + + // THE SUPERSET GUARANTEE (design contract): the ack certifies durable + // AND searchable (the target's snapshot published BEFORE the ack), and + // the source committed after the ack — the target must already hold + // the complete folded state with NO explicit target refresh here. + // (The composite target has no classic _search path; the final fold + // over its mv_state files IS the read.) + assertGoldenFoldOverTargetState(TARGET); + + // The target's OWN derived format: every target generation carries an + // mv_state file set with FOLDED state (the fold definition ran over + // the shipped rows at the target's refresh). + assertTargetHasFoldedStateFiles(); + } + + public void testShipFailureFailsTheFlushAndRetryHeals() throws Exception { + createSourceIndex(); + // Plain (non-composite) target for THIS test: the invariant under test + // is entirely source-side (ship failure => flush refusal => heal via + // idempotent re-ship) and does not depend on the target's engine. + // Close/reopen of a COMPOSITE index currently livelocks on the shard + // lock (slow native close vs reopen — known composite-engine gap, + // tracked in the KB); staging the outage via close needs the plain + // engine until that is fixed. + createPlainTargetIndex(TARGET); + assertColocated(); + + indexDoc("api", "200", 30); + client().admin().indices().prepareRefresh(SOURCE).get(); + // No target refresh: the ack already certified searchability. + assertEquals(1, client().prepareSearch(TARGET).setSize(0).get().getHits().getTotalHits().value()); + + // Break the invariant's precondition: CLOSE the target, ingest more. + // (Deleting would not fail the ship — bulk auto-creates missing + // indices; a closed index rejects writes deterministically.) + assertAcked(client().admin().indices().prepareClose(TARGET)); + indexDoc("web", "200", 80); + + // The refresh-driven flush must FAIL (ship-before-commit) — the new doc + // must not become committed-on-source while its state is nowhere. + // Refresh is a broadcast action: the shard-level flush failure surfaces + // as failed shards on the response (or, if the engine failure cascades + // first, as a thrown exception — both prove the commit was refused). + try { + org.opensearch.action.support.broadcast.BroadcastResponse refresh = client().admin().indices().prepareRefresh(SOURCE).get(); + assertTrue( + "refresh must fail while the ship target is missing; got " + refresh.getFailedShards() + " failed shards", + refresh.getFailedShards() > 0 + ); + } catch (Exception expected) { + // Engine failure cascaded before the response — equally a refused commit. + } + + // Heal: reopen the target; retry the source's failed allocation (its + // recovery flush kept failing while the target was closed and used up + // the allocation retries); the retried flush re-ships (deterministic + // doc ids make re-shipping idempotent). + assertAcked(client().admin().indices().prepareOpen(TARGET)); + // The composite engine's close releases native resources slowly; the + // reopened shard can race the old shard lock and burn allocation + // retries (also: the source's own recovery keeps failing until the + // target is up — ship-before-commit through recovery). Keep retrying + // failed allocations until both indices settle. + assertBusy(() -> { + client().admin().cluster().prepareReroute().setRetryFailed(true).get(); + org.opensearch.action.admin.cluster.health.ClusterHealthResponse health = client().admin() + .cluster() + .prepareHealth(TARGET, SOURCE) + .setWaitForGreenStatus() + .setTimeout(org.opensearch.common.unit.TimeValue.timeValueSeconds(5)) + .get(); + assertFalse("cluster must settle green after target reopen", health.isTimedOut()); + }, 90, java.util.concurrent.TimeUnit.SECONDS); + assertBusy(() -> { + try { + client().admin().indices().prepareRefresh(SOURCE).get(); + } catch (Exception e) { + throw new AssertionError("flush still failing after target recreated", e); + } + }); + // No target refresh (superset guarantee holds through the heal too). + assertBusy(() -> { + long docs = client().prepareSearch(TARGET).setSize(0).get().getHits().getTotalHits().value(); + assertTrue("web/200 state must arrive after heal, saw " + docs + " docs", docs >= 1); + }); + } + + /** + * One finalized state batch, TWO targets: the ref-counted handoff shares + * the same Arrow buffers across both ships — no destination frees the + * batch under the other (a refcount bug surfaces as a use-after-free, + * a double-free IllegalState, or an allocator leak — the test JVM runs + * with arrow.memory.debug.allocator). Both targets must independently + * fold to the exact goldens. + */ + public void testOneBatchShipsToMultipleTargets() throws Exception { + String target2 = "mv_payments_2"; + createSourceIndexWithTargets(TARGET, target2); + createTargetIndex(TARGET); + createTargetIndex(target2); + assertColocated(); + + seedGoldenSegments(); + + assertGoldenFoldOverTargetState(TARGET); + assertGoldenFoldOverTargetState(target2); + } + + /** + * Final fold over the target's OWN mv_state files == the goldens. This is + * the composite target's read (production: the analytics engine path). + */ + private void assertGoldenFoldOverTargetState(String targetIndex) throws Exception { + java.util.List files = targetStateFiles(targetIndex); + assertFalse("target [" + targetIndex + "] must have mv_state files", files.isEmpty()); + String result = MVNativeBridge.searchV2(files, MVConstants.TARGET_FOLD_SEARCH_SQL); + assertEquals( + "api\t200\t4\t115\t10\t50\n" + "api\t500\t1\t900\t900\t900\n" + "batch\t200\t1\t60\t60\t60\n" + "web\t200\t2\t120\t40\t80\n", + result + ); + } + + private java.util.List targetStateFiles(String targetIndex) throws Exception { + java.util.List files = new java.util.ArrayList<>(); + try ( + org.opensearch.common.concurrent.GatedCloseable ref = targetEngine( + targetIndex + ).acquireSnapshot() + ) { + for (org.opensearch.index.engine.exec.WriterFileSet set : ref.get().getSearchableFiles(MVStateDataFormat.NAME)) { + for (String f : set.files()) { + files.add(java.nio.file.Path.of(set.directory()).resolve(f).toString()); + } + } + } + return files; + } + + private org.opensearch.index.engine.DataFormatAwareEngine targetEngine(String targetIndex) { + String nodeName = getClusterState().nodes() + .get(getClusterState().routingTable().index(targetIndex).shard(0).primaryShard().currentNodeId()) + .getName(); + org.opensearch.indices.IndicesService indicesService = internalCluster().getInstance( + org.opensearch.indices.IndicesService.class, + nodeName + ); + org.opensearch.index.shard.IndexShard shard = indicesService.indexServiceSafe( + getClusterState().metadata().index(targetIndex).getIndex() + ).getShard(0); + return (org.opensearch.index.engine.DataFormatAwareEngine) org.opensearch.index.shard.IndexShardTestCase.getIndexer(shard); + } + + /** + * Asserts the target's catalog snapshot carries mv_state file sets — the + * target's derived format materialized folded state per generation. + */ + private void assertTargetHasFoldedStateFiles() throws Exception { + try ( + org.opensearch.common.concurrent.GatedCloseable ref = targetEngine( + TARGET + ).acquireSnapshot() + ) { + java.util.Collection stateSets = ref.get() + .getSearchableFiles(MVStateDataFormat.NAME); + assertFalse("target must carry mv_state file sets (folded state per generation)", stateSets.isEmpty()); + long foldedRows = stateSets.stream().mapToLong(org.opensearch.index.engine.exec.WriterFileSet::numRows).sum(); + // Each target generation folds ITS shipped rows: gen1 folds 3 source- + // gen-1 rows -> 3 groups; gen2 folds 3 source-gen-2 rows -> 3 groups. + // (Cross-generation folding happens at target MERGE — disabled here.) + assertEquals("folded state rows across target generations", 6, foldedRows); + } + } + + /** Golden dataset: 2 segments, 8 docs -> 6 state rows (3 per generation). */ + private void seedGoldenSegments() { + indexDoc("api", "200", 30); + indexDoc("api", "200", 50); + indexDoc("web", "200", 40); + indexDoc("api", "500", 900); + indexDoc("api", "200", 25); + client().admin().indices().prepareRefresh(SOURCE).get(); + indexDoc("api", "200", 10); + indexDoc("web", "200", 80); + indexDoc("batch", "200", 60); + client().admin().indices().prepareRefresh(SOURCE).get(); + } + + // ── Infrastructure ────────────────────────────────────────────────────── + + private void createSourceIndex() { + createSourceIndexWithTargets(TARGET); + } + + private void createSourceIndexWithTargets(String... targets) { + client().admin() + .indices() + .prepareCreate(SOURCE) + .setSettings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.pluggable.dataformat.enabled", true) + .put("index.pluggable.dataformat", "composite") + .put("index.composite.primary_data_format", "parquet") + .putList("index.composite.secondary_data_formats", "lucene", "materialized_view") + .put("index.composite.merge_on_refresh_max_size", "0b") + .putList(MVConstants.SHIP_TARGETS_SETTING, targets) + ) + .setMapping("service", "type=keyword", "status", "type=keyword", "latency_ms", "type=long") + .get(); + ensureGreen(SOURCE); + } + + private void createTargetIndex() { + createTargetIndex(TARGET); + } + + private void createPlainTargetIndex(String name) { + client().admin() + .indices() + .prepareCreate(name) + .setSettings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(MVConstants.COLOCATE_WITH_SETTING, SOURCE) + ) + .setMapping( + "service", + "type=keyword", + "status", + "type=keyword", + "cnt", + "type=long", + "lat_sum", + "type=long", + "lat_min", + "type=long", + "lat_max", + "type=long", + "_mv_source_generation", + "type=long" + ) + .get(); + ensureGreen(name); + } + + private void createTargetIndex(String name) { + // The TARGET is itself a DataFormatAwareEngine composite index: parquet + // primary + lucene (search + capabilities) + the mv_state DERIVED + // format, whose definition is the FOLD of the shipped state schema — + // per target generation the mv_state writer materializes folded, + // group-key-sorted state (the embedded-MV shape on the MV index). + client().admin() + .indices() + .prepareCreate(name) + .setSettings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.pluggable.dataformat.enabled", true) + .put("index.pluggable.dataformat", "composite") + .put("index.composite.primary_data_format", "parquet") + .putList("index.composite.secondary_data_formats", "lucene", "mv_state") + .put("index.composite.merge_on_refresh_max_size", "0b") + .put(MVConstants.COLOCATE_WITH_SETTING, SOURCE) + ) + .setMapping( + "service", + "type=keyword", + "status", + "type=keyword", + "cnt", + "type=long", + "lat_sum", + "type=long", + "lat_min", + "type=long", + "lat_max", + "type=long", + "_mv_source_generation", + "type=long" + ) + .get(); + ensureGreen(name); + } + + /** + * The colocation decider must put the target's primary on the node holding + * the source's primary (ordinal pairing, 2 data nodes make this a real + * constraint rather than a tautology). + */ + private void assertColocated() { + String sourceNode = getClusterState().routingTable().index(SOURCE).shard(0).primaryShard().currentNodeId(); + String targetNode = getClusterState().routingTable().index(TARGET).shard(0).primaryShard().currentNodeId(); + assertEquals("MV target primary must colocate with the source primary", sourceNode, targetNode); + } + + private void indexDoc(String service, String status, long latencyMs) { + IndexResponse r = client().prepareIndex() + .setIndex(SOURCE) + .setSource("service", service, "status", status, "latency_ms", latencyMs) + .get(); + assertEquals(RestStatus.CREATED, r.status()); + } +} diff --git a/sandbox/plugins/mv-data-format/src/internalClusterTest/java/org/opensearch/mv/MVViewsIT.java b/sandbox/plugins/mv-data-format/src/internalClusterTest/java/org/opensearch/mv/MVViewsIT.java new file mode 100644 index 0000000000000..6dbec4d8d21c7 --- /dev/null +++ b/sandbox/plugins/mv-data-format/src/internalClusterTest/java/org/opensearch/mv/MVViewsIT.java @@ -0,0 +1,165 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.mv; + +import com.carrotsearch.randomizedtesting.ThreadFilter; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.opensearch.arrow.allocator.ArrowBasePlugin; +import org.opensearch.be.datafusion.DataFusionPlugin; +import org.opensearch.be.lucene.LucenePlugin; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.network.NetworkModule; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.common.util.concurrent.OpenSearchExecutors; +import org.opensearch.composite.CompositeDataFormatPlugin; +import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.plugins.Plugin; +import org.opensearch.test.MockLogAppender; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.transport.Netty4ModulePlugin; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; +import java.util.regex.Pattern; + +/** + * The `index.mv.views` UX end to end (decisions 20/21/23/24): the user + * creates ONLY the source index with a views declaration — no formats, no + * state schema, no target index — and the system derives the source's MV + * settings ({@link MVViewsService.Provider}), auto-creates the colocated + * target with the state mapping + single hidden provenance field + * ({@link MVViewsService.TargetCreator}), and the ship/fold pipeline runs + * exactly as if both indices had been created by hand. + */ +@ThreadLeakFilters(filters = MVViewsIT.NativeThreadFilter.class) +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 1) +public class MVViewsIT extends OpenSearchIntegTestCase { + + public static class NativeThreadFilter implements ThreadFilter { + private static final Pattern GENERIC = Pattern.compile("^Thread-\\d+$"); + + @Override + public boolean reject(Thread t) { + return GENERIC.matcher(t.getName()).matches(); + } + } + + private static final String SOURCE = "payments"; + /** Generated name: {@code _mv_} (decision 23, unnamed view). */ + private static final String GENERATED_TARGET = "payments_mv_payments"; + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG, true) + .put(NetworkModule.HTTP_TYPE_KEY, Netty4ModulePlugin.NETTY_HTTP_TRANSPORT_NAME) + // 2 processors: ship-before-commit must never share a 1-thread + // write pool with the target's apply (same-pool deadlock — see + // MVSeparateIndexPocIT's note). + .put(OpenSearchExecutors.NODE_PROCESSORS_SETTING.getKey(), 2) + .build(); + } + + @Override + protected Collection> nodePlugins() { + return Arrays.asList( + ArrowBasePlugin.class, + Netty4ModulePlugin.class, + CompositeDataFormatPlugin.class, + ParquetDataFormatPlugin.class, + LucenePlugin.class, + DataFusionPlugin.class, + MVDataFormatPlugin.class, + MVStateDataFormatPlugin.class + ); + } + + public void testViewsDeclarationDrivesTheWholePipeline() throws Exception { + // The ONLY thing the user does: create the source with a views entry. + client().admin() + .indices() + .prepareCreate(SOURCE) + .setSettings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.composite.merge_on_refresh_max_size", "0b") + .putList(MVConstants.VIEWS_SETTING, "payments") + ) + // Mapping INLINE in the create — works since the provider-collision + // fix (the composite plugin's provider defers to MV-derived formats). + .setMapping("service", "type=keyword", "status", "type=keyword", "latency_ms", "type=long") + .get(); + ensureGreen(SOURCE); + + // Provider must have derived the composite+MV source settings. + Settings sourceSettings = client().admin().indices().prepareGetSettings(SOURCE).get().getIndexToSettings().get(SOURCE); + assertEquals("composite", sourceSettings.get("index.pluggable.dataformat")); + assertTrue( + "materialized_view must be derived into the formats", + sourceSettings.getAsList("index.composite.secondary_data_formats").contains("materialized_view") + ); + assertEquals(java.util.List.of(GENERATED_TARGET), sourceSettings.getAsList(MVConstants.SHIP_TARGETS_SETTING)); + + // Target must be auto-created (cluster-manager listener) — with the + // derived state mapping and colocation. + assertBusy(() -> { + assertTrue( + "target must be auto-created", + client().admin().cluster().prepareState().get().getState().metadata().hasIndex(GENERATED_TARGET) + ); + }); + ensureGreen(GENERATED_TARGET); + Settings targetSettings = client().admin() + .indices() + .prepareGetSettings(GENERATED_TARGET) + .get() + .getIndexToSettings() + .get(GENERATED_TARGET); + assertEquals(SOURCE, targetSettings.get(MVConstants.COLOCATE_WITH_SETTING)); + assertTrue(targetSettings.getAsList("index.composite.secondary_data_formats").contains("mv_state")); + Map mapping = client().admin() + .indices() + .prepareGetMappings(GENERATED_TARGET) + .get() + .getMappings() + .get(GENERATED_TARGET) + .sourceAsMap(); + @SuppressWarnings("unchecked") + Map properties = (Map) mapping.get("properties"); + assertTrue("state schema derived", properties.containsKey("cnt") && properties.containsKey("lat_sum")); + assertTrue("single hidden provenance field (decision 21)", properties.containsKey("_mv_source_generation")); + assertFalse("dropped provenance fields must be gone", properties.containsKey("_mv_source_index")); + + // And the pipeline actually runs: ingest -> refresh -> state shipped + // to the AUTO-CREATED target with a durable+searchable ack. (The + // composite target has no classic _search path; the ship-ack log is + // the same proof the main POC IT uses.) + try (MockLogAppender appender = MockLogAppender.createForLoggers(LogManager.getLogger("org.opensearch.mv.MVStateShipper"))) { + appender.addExpectation( + new MockLogAppender.SeenEventExpectation( + "state shipped to the auto-created target", + "org.opensearch.mv.MVStateShipper", + Level.INFO, + "*-> [" + GENERATED_TARGET + "][0] (acked durable+searchable)*" + ) + ); + client().prepareIndex(SOURCE).setSource("service", "api", "status", "200", "latency_ms", 30).get(); + client().prepareIndex(SOURCE).setSource("service", "api", "status", "200", "latency_ms", 70).get(); + client().admin().indices().prepareRefresh(SOURCE).get(); + appender.assertAllExpectationsMatched(); + } + } +} diff --git a/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVColocationAllocationDecider.java b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVColocationAllocationDecider.java new file mode 100644 index 0000000000000..0d4f78cdb8ef7 --- /dev/null +++ b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVColocationAllocationDecider.java @@ -0,0 +1,91 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.mv; + +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.routing.RoutingNode; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.allocation.RoutingAllocation; +import org.opensearch.cluster.routing.allocation.decider.AllocationDecider; +import org.opensearch.cluster.routing.allocation.decider.Decision; +import org.opensearch.core.index.shard.ShardId; + +/** + * Colocates a separate-index MV target's primaries with the source index's + * primaries, shard ordinal to shard ordinal (deterministic 1:1 pairing — + * equal shard counts are validated at MV creation). + * + *

The target index opts in via {@code index.mv.colocate_with = }. + * For a target PRIMARY of ordinal i: {@code canAllocate} answers YES only on + * the node holding the source's ACTIVE primary of ordinal i (THROTTLE while + * that primary is unassigned — retry rather than wedge); {@code canRemain} + * answers NO once the source primary has moved, so the standard reactive + * machinery relocates the target to follow (no custom mover). Replicas are + * unconstrained: only the write handoff needs primary-primary locality. + * + *

Unlike {@code ResizeAllocationDecider} (one-shot, initial recovery only), + * this pairing is PERSISTENT — the decider fires for the target's whole life. + * Colocation is an optimization for the in-process state handoff, not a + * correctness requirement: while the pair is split (failover window), the + * ship path falls back to transport with identical ack semantics. See + * separate-index/technical-challenges.md §1–§3. + */ +public final class MVColocationAllocationDecider extends AllocationDecider { + + public static final String NAME = "mv_colocation"; + + @Override + public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + return decide(shardRouting, node, allocation, "cannot allocate"); + } + + @Override + public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + return decide(shardRouting, node, allocation, "cannot remain"); + } + + private Decision decide(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation, String verb) { + if (shardRouting.primary() == false) { + return allocation.decision(Decision.YES, NAME, "replicas are not colocation-constrained"); + } + IndexMetadata targetMetadata = allocation.metadata().index(shardRouting.index()); + if (targetMetadata == null) { + return allocation.decision(Decision.YES, NAME, "no index metadata"); + } + String sourceIndexName = targetMetadata.getSettings().get(MVConstants.COLOCATE_WITH_SETTING); + if (sourceIndexName == null || sourceIndexName.isEmpty()) { + return allocation.decision(Decision.YES, NAME, "index is not an MV colocation target"); + } + IndexMetadata sourceMetadata = allocation.metadata().index(sourceIndexName); + if (sourceMetadata == null) { + // Source deleted — do not wedge the target's allocation on a + // dangling pairing; lifecycle handling decides the target's fate. + return allocation.decision(Decision.YES, NAME, "colocation source [" + sourceIndexName + "] does not exist"); + } + ShardId sourceShardId = new ShardId(sourceMetadata.getIndex(), shardRouting.id()); + ShardRouting sourcePrimary = allocation.routingNodes().activePrimary(sourceShardId); + if (sourcePrimary == null || sourcePrimary.currentNodeId() == null) { + // Source primary unassigned: allow allocation ANYWHERE rather than + // wait. Waiting creates a circular dependency — the source's own + // recovery flush ships to this target (ship-before-commit), so a + // target that waits for the source can deadlock the pair after a + // joint outage. Availability wins; once the source is active again, + // canRemain flips NO and reactive following restores colocation. + return allocation.decision(Decision.YES, NAME, "source primary " + sourceShardId + " is unassigned; not constraining"); + } + if (sourcePrimary.currentNodeId().equals(node.nodeId())) { + return allocation.decision(Decision.YES, NAME, "node holds the source primary " + sourceShardId); + } + return allocation.decision( + Decision.NO, + NAME, + verb + ": source primary " + sourceShardId + " is on node [" + sourcePrimary.currentNodeId() + "], not [" + node.nodeId() + "]" + ); + } +} diff --git a/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVCommitSyncAction.java b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVCommitSyncAction.java new file mode 100644 index 0000000000000..b30fd4392ffcb --- /dev/null +++ b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVCommitSyncAction.java @@ -0,0 +1,103 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.mv; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.ActionType; +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import java.io.IOException; + +/** + * Commit sync (decision 25), the second half of ship-before-commit's + * superset rule: before the SOURCE commits generation N, the TARGET must + * durably commit a catalog snapshot at least as new as the one the gen-N + * ship ack reported. Target committed state may lead the source's, never + * trail it. + * + *

Same locality contract as {@link MVShipStateAction}: colocation pins + * the pair to one node, dispatch is in-JVM (NodeClient), and + * {@link Request#writeTo} throws as the locality tripwire. + */ +public final class MVCommitSyncAction extends ActionType { + + public static final String NAME = "indices:data/write/mv_commit_sync"; + public static final MVCommitSyncAction INSTANCE = new MVCommitSyncAction(); + + private MVCommitSyncAction() { + super(NAME, Response::new); + } + + /** Commit request for one target shard: commit at least {@code minVersion}. */ + public static class Request extends ActionRequest { + private final String targetIndex; + private final int targetShard; + private final long minVersion; + + public Request(String targetIndex, int targetShard, long minVersion) { + this.targetIndex = targetIndex; + this.targetShard = targetShard; + this.minVersion = minVersion; + } + + public Request(StreamInput in) throws IOException { + super(in); + throw new IllegalStateException("mv commit sync is local-only (hard locality rule) — must never deserialize"); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + // Locality tripwire: serializing this request means the pair is + // NOT colocated — a bug, not a fallback (hard locality rule). + throw new IllegalStateException("mv commit sync is local-only (hard locality rule) — must never serialize"); + } + + public String targetIndex() { + return targetIndex; + } + + public int targetShard() { + return targetShard; + } + + public long minVersion() { + return minVersion; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + } + + /** Ack: the target's committed catalog snapshot version (>= requested min). */ + public static class Response extends ActionResponse { + private final long committedVersion; + + public Response(long committedVersion) { + this.committedVersion = committedVersion; + } + + public Response(StreamInput in) throws IOException { + this.committedVersion = in.readVLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(committedVersion); + } + + public long committedVersion() { + return committedVersion; + } + } +} diff --git a/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVCommitSyncTransportHandler.java b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVCommitSyncTransportHandler.java new file mode 100644 index 0000000000000..76a21d9737f58 --- /dev/null +++ b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVCommitSyncTransportHandler.java @@ -0,0 +1,119 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.mv; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.admin.indices.flush.FlushRequest; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.common.inject.Inject; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.shard.IndexShard; +import org.opensearch.indices.IndicesService; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +/** + * Applies a commit-sync request on the LOCAL target primary (decision 25): + * flushes the target shard — committing its latest catalog snapshot, which + * is at least the version the preceding ship acks reported — and responds + * with the committed version. Runs synchronously on the calling thread + * (SAME executor): the caller IS the source's flush thread inside its + * commit section, and the target flush must complete before the source + * commit proceeds; a pool hop would only add deadlock surface (the + * write-pool lesson from ship-before-commit). + * + *

Same hard locality rule as the ship handler: the pair is colocated, + * a non-local target primary fails the request (and with it the source's + * flush) rather than falling back to remote. + */ +public final class MVCommitSyncTransportHandler extends HandledTransportAction { + + private static final Logger logger = LogManager.getLogger(MVCommitSyncTransportHandler.class); + + private final IndicesService indicesService; + private final org.opensearch.cluster.service.ClusterService clusterService; + + @Inject + public MVCommitSyncTransportHandler( + TransportService transportService, + ActionFilters actionFilters, + IndicesService indicesService, + org.opensearch.cluster.service.ClusterService clusterService + ) { + super(MVCommitSyncAction.NAME, transportService, actionFilters, MVCommitSyncAction.Request::new, ThreadPool.Names.SAME); + this.indicesService = indicesService; + this.clusterService = clusterService; + } + + @Override + protected void doExecute(Task task, MVCommitSyncAction.Request request, ActionListener listener) { + try { + org.opensearch.cluster.routing.ShardRouting primary = clusterService.state() + .routingTable() + .index(request.targetIndex()) + .shard(request.targetShard()) + .primaryShard(); + if (primary == null || primary.active() == false || primary.currentNodeId() == null) { + listener.onFailure( + new IllegalStateException( + "mv commit sync: target primary [" + request.targetIndex() + "][" + request.targetShard() + "] is not active" + ) + ); + return; + } + if (primary.currentNodeId().equals(clusterService.localNode().getId()) == false) { + // Hard locality rule — split pair refuses the source commit. + listener.onFailure( + new IllegalStateException( + "mv commit sync: target primary [" + + request.targetIndex() + + "][" + + request.targetShard() + + "] is not local (colocation violated) — refusing the source commit" + ) + ); + return; + } + ShardId shardId = new ShardId(clusterService.state().metadata().index(request.targetIndex()).getIndex(), request.targetShard()); + IndexShard shard = indicesService.indexServiceSafe(shardId.getIndex()).getShard(shardId.id()); + // Commit the target's latest catalog snapshot. The engine skips + // the write when the snapshot id is already committed — repeated + // source flushes with no new generations stay cheap. + shard.flush(new FlushRequest().force(false).waitIfOngoing(true)); + long committed = shard.compositeCatalogSnapshotVersion(); + if (committed >= 0 && committed < request.minVersion()) { + listener.onFailure( + new IllegalStateException( + "mv commit sync: target [" + + request.targetIndex() + + "] committed version " + + committed + + " < required " + + request.minVersion() + ) + ); + return; + } + logger.debug( + "mv commit sync: [{}][{}] committed catalog version {} (required >= {})", + request.targetIndex(), + request.targetShard(), + committed, + request.minVersion() + ); + listener.onResponse(new MVCommitSyncAction.Response(committed)); + } catch (Exception e) { + listener.onFailure(e); + } + } +} diff --git a/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVConstants.java b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVConstants.java new file mode 100644 index 0000000000000..2e97ec9018dd2 --- /dev/null +++ b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVConstants.java @@ -0,0 +1,123 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.mv; + +import java.util.List; + +/** + * POC hardcoded constants — the single fixed materialized view (v2: + * multi-key, multi-agg). + * + *

Definition: {@code SELECT service, status, COUNT(*), SUM(latency_ms), + * MIN(latency_ms), MAX(latency_ms) FROM mv_input GROUP BY service, status}. + * The table name inside the SQL is always {@code mv_input} — the native + * writer registers the fed batches under that name. + * + *

State-file schema comes FROM THE PLAN (state-suffixed columns); Java + * only knows the group keys and the search template. + */ +public final class MVConstants { + + /** Canonical table name every definition SQL is written against. */ + public static final String INPUT_TABLE = "mv_input"; + + private MVConstants() {} + + /** The MV definition executed by the native writer over fed batches. */ + public static final String MV_SQL = "SELECT service, status, COUNT(*), SUM(latency_ms), MIN(latency_ms), MAX(latency_ms) " + + "FROM mv_input GROUP BY service, status"; + + /** Group-by columns (leading state-file columns; also the sort key). */ + public static final List GROUP_KEYS = List.of("service", "status"); + + /** Metric column captured alongside the keys. */ + public static final String METRIC_FIELD = "latency_ms"; + + /** + * Search template over state files: Final-fold of the state columns. + * {@code __MV_STATES__} is replaced natively with the UNION ALL of the + * snapshot's state files. State column names are DataFusion's + * format_state_name output for the MV_SQL aggregates. + */ + public static final String SEARCH_SQL = "SELECT service, status, " + + "SUM(\"count(Int64(1))[count]\") AS cnt, " + + "SUM(\"sum(mv_input.latency_ms)[sum]\") AS lat_sum, " + + "MIN(\"min(mv_input.latency_ms)[value]\") AS lat_min, " + + "MAX(\"max(mv_input.latency_ms)[value]\") AS lat_max " + + "FROM __MV_STATES__ GROUP BY service, status ORDER BY service, status"; + + /** Directory name under the shard data path; also the format name. */ + public static final String DIR = MVDataFormat.NAME; + + /** + * Raw state export over a finalized state file (separate-index ship path): + * SELECT of the state columns verbatim — no folding, the MV index folds on + * read. Order matches {@link #SHIP_FIELDS}. + */ + public static final String EXPORT_SQL = "SELECT service, status, " + + "\"count(Int64(1))[count]\", " + + "\"sum(mv_input.latency_ms)[sum]\", " + + "\"min(mv_input.latency_ms)[value]\", " + + "\"max(mv_input.latency_ms)[value]\" " + + "FROM __MV_STATES__ ORDER BY service, status"; + + /** MV-index document field names, parallel to {@link #EXPORT_SQL} columns. */ + public static final List SHIP_FIELDS = List.of("service", "status", "cnt", "lat_sum", "lat_min", "lat_max"); + + /** + * Source index setting listing the target MV indices; non-empty enables the + * ship path. Multiple targets share ONE finalized state batch via + * {@link MVRefCountedStateBatch} — the flush commits only when EVERY + * target has acked (the invariant holds per target). + */ + public static final String SHIP_TARGETS_SETTING = "index.mv.ship_targets"; + + /** + * User-facing MV declaration (decisions 20/22/23): list of + * {@code definition} or {@code definition:targetName} entries on the + * SOURCE index. Everything else (formats, ship targets, the target index + * itself) is derived — see {@link MVViewsService}. + */ + public static final String VIEWS_SETTING = "index.mv.views"; + + /** + * Validation-scoped read gate (dynamic, on the MV TARGET index): when + * true, analytics scans of this index serve the catalog snapshot's + * mv_state Arrow files as the fragment's PARTIAL output (strict — any + * misalignment throws). The query keeps the ORIGINAL definition shape; + * the coordinator Final computes final answers (incl. avg) natively. + */ + public static final String SERVE_STATE_SETTING = "index.mv.serve_state"; + + /** Index setting naming the MV definition (POC named-spec registry). */ + public static final String DEFINITION_SETTING = "index.mv.definition"; + + /** + * Target index setting naming the SOURCE index whose primaries this + * index's primaries must colocate with (ordinal 1:1 pairing). Consumed by + * {@link MVColocationAllocationDecider}. + */ + public static final String COLOCATE_WITH_SETTING = "index.mv.colocate_with"; + + /** + * Final fold over the TARGET's mv_state files (state column names from the + * TARGET_FOLD definition, DataFusion format_state_name output). + */ + public static final String TARGET_FOLD_SEARCH_SQL = "SELECT service, status, " + + "SUM(\"sum(mv_input.cnt)[sum]\") AS cnt, " + + "SUM(\"sum(mv_input.lat_sum)[sum]\") AS lat_sum, " + + "MIN(\"min(mv_input.lat_min)[value]\") AS lat_min, " + + "MAX(\"max(mv_input.lat_max)[value]\") AS lat_max " + + "FROM __MV_STATES__ GROUP BY service, status ORDER BY service, status"; + + /** MV state file name for a writer generation (Arrow IPC — decision 17). */ + public static String mvFileName(long writerGeneration) { + return "_mv_poc_" + Long.toHexString(writerGeneration) + ".mv.arrow"; + } +} diff --git a/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVDataFormat.java b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVDataFormat.java new file mode 100644 index 0000000000000..e0b1330ae8bec --- /dev/null +++ b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVDataFormat.java @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.mv; + +import org.opensearch.index.engine.dataformat.DerivedDataFormat; + +/** + * POC materialized-view data format on a SOURCE index: a {@link + * DerivedDataFormat} whose per-segment output is aggregate state computed + * from the ingest broadcast (embedded mode) or shipped to a separate MV + * index before commit (ship mode). The derived-format contract (row-parity + * exempt, may emit no files, never claims fields) comes from the base type. + */ +public final class MVDataFormat extends DerivedDataFormat { + + public static final String NAME = "materialized_view"; + public static final MVDataFormat INSTANCE = new MVDataFormat(); + + private MVDataFormat() {} + + @Override + public String name() { + return NAME; + } +} diff --git a/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVDataFormatPlugin.java b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVDataFormatPlugin.java new file mode 100644 index 0000000000000..53e4d48d8ba66 --- /dev/null +++ b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVDataFormatPlugin.java @@ -0,0 +1,187 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.mv; + +import org.opensearch.index.IndexSettings; +import org.opensearch.index.engine.dataformat.DataFormat; +import org.opensearch.index.engine.dataformat.DataFormatDescriptor; +import org.opensearch.index.engine.dataformat.DataFormatPlugin; +import org.opensearch.index.engine.dataformat.DataFormatRegistry; +import org.opensearch.index.engine.dataformat.IndexingEngineConfig; +import org.opensearch.index.engine.dataformat.IndexingExecutionEngine; +import org.opensearch.index.engine.dataformat.ReaderManagerConfig; +import org.opensearch.index.engine.exec.EngineReaderManager; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.store.checksum.GenericCRC32ChecksumHandler; +import org.opensearch.plugins.ActionPlugin.ActionHandler; +import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.SearchBackEndPlugin; + +import java.util.Map; +import java.util.function.Supplier; + +/** + * POC(mv): the materialized-view data format plugin. Registers the derived + * "materialized_view" format; indices opt in via + * {@code index.composite.secondary_data_formats: ["materialized_view"]}. + */ +public class MVDataFormatPlugin extends Plugin + implements + DataFormatPlugin, + SearchBackEndPlugin, + org.opensearch.plugins.ClusterPlugin, + org.opensearch.plugins.ActionPlugin, + org.opensearch.plugins.ExtensiblePlugin { + + /** + * Node client captured at component creation; used by the separate-index + * ship path (POC wiring — production ships over a dedicated transport + * action with an ack listener rather than a blocking client bulk). + */ + private volatile org.opensearch.transport.client.Client client; + private volatile org.opensearch.cluster.service.ClusterService clusterService; + + public MVDataFormatPlugin() {} + + @Override + public java.util.Collection createComponents( + org.opensearch.transport.client.Client client, + org.opensearch.cluster.service.ClusterService clusterService, + org.opensearch.threadpool.ThreadPool threadPool, + org.opensearch.watcher.ResourceWatcherService resourceWatcherService, + org.opensearch.script.ScriptService scriptService, + org.opensearch.core.xcontent.NamedXContentRegistry xContentRegistry, + org.opensearch.env.Environment environment, + org.opensearch.env.NodeEnvironment nodeEnvironment, + org.opensearch.core.common.io.stream.NamedWriteableRegistry namedWriteableRegistry, + org.opensearch.cluster.metadata.IndexNameExpressionResolver indexNameExpressionResolver, + java.util.function.Supplier repositoriesServiceSupplier + ) { + this.client = client; + this.clusterService = clusterService; + // D20/D23/D24: auto-create MV target indices for sources declaring + // index.mv.views (cluster-manager only; tolerant of re-entry). + clusterService.addListener(new MVViewsService.TargetCreator(client)); + // NOTE (real-node deployment): all native-using plugins must share ONE + // native instance — deploy with -Dnative.lib.path pointing at a single + // .so (SymbolLookup.libraryLookup dlopens the same handle; globals + // shared; the DataFusion plugin's start() initializes the runtime + // manager for everyone). Without the property, each classloader + // extracts its OWN temp copy of the embedded lib = separate globals = + // "Runtime manager not initialized" from every plugin that didn't + // init its own. Do NOT init here: double-init corrupts the shared + // manager. + return java.util.List.of(); + } + + @Override + public java.util.List> getSettings() { + return java.util.List.of( + org.opensearch.common.settings.Setting.listSetting( + MVConstants.SHIP_TARGETS_SETTING, + java.util.List.of(), + java.util.function.Function.identity(), + org.opensearch.common.settings.Setting.Property.IndexScope + ), + org.opensearch.common.settings.Setting.simpleString( + MVConstants.COLOCATE_WITH_SETTING, + org.opensearch.common.settings.Setting.Property.IndexScope + ), + org.opensearch.common.settings.Setting.simpleString( + MVConstants.DEFINITION_SETTING, + "payments", + org.opensearch.common.settings.Setting.Property.IndexScope + ), + org.opensearch.common.settings.Setting.boolSetting( + MVConstants.SERVE_STATE_SETTING, + false, + org.opensearch.common.settings.Setting.Property.IndexScope, + org.opensearch.common.settings.Setting.Property.Dynamic + ), + org.opensearch.common.settings.Setting.listSetting( + MVConstants.VIEWS_SETTING, + java.util.List.of(), + java.util.function.Function.identity(), + org.opensearch.common.settings.Setting.Property.IndexScope + ) + ); + } + + @Override + public java.util.Collection getAdditionalIndexSettingProviders() { + return java.util.List.of(new MVViewsService.Provider()); + } + + @Override + public java.util.Collection createAllocationDeciders( + org.opensearch.common.settings.Settings settings, + org.opensearch.common.settings.ClusterSettings clusterSettings + ) { + return java.util.List.of(new MVColocationAllocationDecider()); + } + + @Override + public + java.util.List> + getActions() { + return java.util.List.of( + new ActionHandler<>(MVShipStateAction.INSTANCE, MVShipStateTransportHandler.class), + new ActionHandler<>(MVCommitSyncAction.INSTANCE, MVCommitSyncTransportHandler.class) + ); + } + + @Override + public DataFormat getDataFormat() { + return MVDataFormat.INSTANCE; + } + + @Override + public IndexingExecutionEngine indexingEngine(IndexingEngineConfig config) { + java.util.List shipTargets = config.indexSettings().getSettings().getAsList(MVConstants.SHIP_TARGETS_SETTING); + String definition = config.indexSettings().getSettings().get(MVConstants.DEFINITION_SETTING, "payments"); + return new MVIndexingEngine( + config.store().shardPath(), + config.indexSettings().getIndex().getName(), + MVDefinitionSpec.source(definition), + MVDataFormat.INSTANCE, + shipTargets == null ? java.util.List.of() : shipTargets, + () -> client, + () -> clusterService + ); + } + + @Override + public Map> getFormatDescriptors(IndexSettings indexSettings, DataFormatRegistry registry) { + return Map.of(MVDataFormat.NAME, () -> new DataFormatDescriptor(MVDataFormat.NAME, new GenericCRC32ChecksumHandler())); + } + + @Override + public void assignCapabilities(MappedFieldType fieldType, IndexSettings indexSettings, DataFormatRegistry dataFormatRegistry) { + // Derived format: claims no field capabilities ever. Setting an empty + // map here would clobber assignments made by other formats — so do + // nothing at all. + } + + // ---- SearchBackEndPlugin (reader lifecycle for the materialized_view format) ---- + + @Override + public String name() { + return MVDataFormat.NAME; + } + + @Override + public java.util.List getSupportedFormats() { + return java.util.List.of(MVDataFormat.NAME); + } + + @Override + public EngineReaderManager createReaderManager(ReaderManagerConfig settings) { + return new MVReaderManager(); + } +} diff --git a/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVDefinitionSpec.java b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVDefinitionSpec.java new file mode 100644 index 0000000000000..cf347066e82d2 --- /dev/null +++ b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVDefinitionSpec.java @@ -0,0 +1,172 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.mv; + +import java.util.List; + +/** + * POC(mv): one MV definition in the form the write path needs — the columns + * to capture from the composite broadcast (in buffer/state order: group keys + * first) and the Partial-stopped definition SQL the native writer maintains. + * + *

Two hardcoded instances exist until MV metadata lands: + *

    + *
  • {@link #SOURCE} — the source index's definition over raw fields.
    • + *
  • {@link #TARGET_FOLD} — the separate-index TARGET's definition: the + * FOLD of the source's state schema. Shipped state rows arrive as + * documents; folding them (SUM of count/sum states, MIN/MAX of extrema + * states) is aggregation-of-state, so the target is simply a composite + * index whose derived format maintains folded per-segment state — the + * embedded-MV shape, applied to the MV index itself.
    • + *
+ * + * @param columns captured columns, group keys first (buffer/state order) + * @param groupKeys number of leading group-key columns + * @param sql definition SQL over table {@code mv_input}, maintained + * Partial-stopped by the native writer + * @param shipFields target-index document field names for the shipped state + * rows, positional with the finalized state batch's columns + * (group keys first, then one field per aggregate state) + */ +public record MVDefinitionSpec(List columns, int groupKeys, String sql, List shipFields) { + + /** Column types the POC forward buffer supports. */ + public enum ColumnType { + UTF8, + INT64 + } + + /** One captured column: broadcast field name + buffer vector type. */ + public record Column(String name, ColumnType type) { + } + + public MVDefinitionSpec { + columns = List.copyOf(columns); + shipFields = List.copyOf(shipFields); + } + + /** + * Named spec lookup (POC stand-in for MV metadata): sources resolve + * {@code index.mv.definition}; targets resolve the same name's fold. + */ + public static MVDefinitionSpec source(String name) { + return switch (name) { + case "payments" -> SOURCE; + case "clickbench_q9" -> CLICKBENCH_Q9; + case "clickbench_q9_native" -> CLICKBENCH_Q9_NATIVE; + default -> throw new IllegalArgumentException("unknown mv definition [" + name + "]"); + }; + } + + public static MVDefinitionSpec fold(String name) { + return switch (name) { + case "payments" -> TARGET_FOLD; + case "clickbench_q9" -> CLICKBENCH_Q9_FOLD; + case "clickbench_q9_native" -> CLICKBENCH_Q9_NATIVE_FOLD; + default -> throw new IllegalArgumentException("unknown mv fold definition [" + name + "]"); + }; + } + + /** The source index's definition (raw fields → state). */ + public static final MVDefinitionSpec SOURCE = new MVDefinitionSpec( + List.of(new Column("service", ColumnType.UTF8), new Column("status", ColumnType.UTF8), new Column("latency_ms", ColumnType.INT64)), + 2, + MVConstants.MV_SQL, + List.of("service", "status", "cnt", "lat_sum", "lat_min", "lat_max") + ); + + /** The target index's definition: the FOLD of the shipped state schema. */ + public static final MVDefinitionSpec TARGET_FOLD = new MVDefinitionSpec( + List.of( + new Column("service", ColumnType.UTF8), + new Column("status", ColumnType.UTF8), + new Column("cnt", ColumnType.INT64), + new Column("lat_sum", ColumnType.INT64), + new Column("lat_min", ColumnType.INT64), + new Column("lat_max", ColumnType.INT64) + ), + 2, + "SELECT service, status, SUM(cnt), SUM(lat_sum), MIN(lat_min), MAX(lat_max) FROM mv_input GROUP BY service, status", + List.of("service", "status", "cnt", "lat_sum", "lat_min", "lat_max") + ); + + /** + * ClickBench q9's mergeable core over the {@code hits} mapping (KB: + * clickbench-reference): integer group key, short metrics. AVG is NOT in + * the definition — it decomposes: the read computes + * {@code AVG(ResolutionWidth) = SUM-state / COUNT-state} exactly. + * Reference query: {@code SELECT RegionID, SUM(AdvEngineID), COUNT(*), + * AVG(ResolutionWidth) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10}. + */ + public static final MVDefinitionSpec CLICKBENCH_Q9 = new MVDefinitionSpec( + List.of( + new Column("RegionID", ColumnType.INT64), + new Column("AdvEngineID", ColumnType.INT64), + new Column("ResolutionWidth", ColumnType.INT64) + ), + 1, + "SELECT \"RegionID\", COUNT(*), SUM(\"AdvEngineID\"), SUM(\"ResolutionWidth\"), MIN(\"ResolutionWidth\"), MAX(\"ResolutionWidth\") " + + "FROM mv_input GROUP BY \"RegionID\"", + List.of("RegionID", "cnt", "adv_sum", "res_sum", "res_min", "res_max") + ); + + /** Fold of {@link #CLICKBENCH_Q9}'s state schema on the target. */ + public static final MVDefinitionSpec CLICKBENCH_Q9_FOLD = new MVDefinitionSpec( + List.of( + new Column("RegionID", ColumnType.INT64), + new Column("cnt", ColumnType.INT64), + new Column("adv_sum", ColumnType.INT64), + new Column("res_sum", ColumnType.INT64), + new Column("res_min", ColumnType.INT64), + new Column("res_max", ColumnType.INT64) + ), + 1, + "SELECT \"RegionID\", SUM(cnt), SUM(adv_sum), SUM(res_sum), MIN(res_min), MAX(res_max) FROM mv_input GROUP BY \"RegionID\"", + List.of("RegionID", "cnt", "adv_sum", "res_sum", "res_min", "res_max") + ); + + /** + * ZERO-TRANSLATION q9 (native read validation): the definition IS the + * query — AVG kept intact so the state file carries DataFusion's own avg + * state pair [count, sum] in the query's exact partial column order: + * [RegionID, sum(AdvEngineID)[sum], count(*)[count], avg[count], avg[sum]]. + * A strict read then serves these files AS the fragment's Partial output + * and the coordinator Final merges + evaluates (divide-once) natively. + */ + public static final MVDefinitionSpec CLICKBENCH_Q9_NATIVE = new MVDefinitionSpec( + List.of( + new Column("RegionID", ColumnType.INT64), + new Column("AdvEngineID", ColumnType.INT64), + new Column("ResolutionWidth", ColumnType.INT64) + ), + 1, + "SELECT \"RegionID\", SUM(\"AdvEngineID\"), COUNT(*), AVG(\"ResolutionWidth\") FROM mv_input GROUP BY \"RegionID\"", + List.of("RegionID", "adv_sum", "cnt", "avg_cnt", "avg_sum") + ); + + /** + * Fold of {@link #CLICKBENCH_Q9_NATIVE}'s state on the target. The + * UNSIGNED cast keeps the folded avg-count column bit-identical to + * DataFusion's avg state type (UInt64) — the strict read compares + * positional types EXACTLY and throws on any drift (crude by design + * for the validation phase). + */ + public static final MVDefinitionSpec CLICKBENCH_Q9_NATIVE_FOLD = new MVDefinitionSpec( + List.of( + new Column("RegionID", ColumnType.INT64), + new Column("adv_sum", ColumnType.INT64), + new Column("cnt", ColumnType.INT64), + new Column("avg_cnt", ColumnType.INT64), + new Column("avg_sum", ColumnType.INT64) + ), + 1, + "SELECT \"RegionID\", SUM(adv_sum), SUM(cnt), SUM(CAST(avg_cnt AS BIGINT UNSIGNED)), SUM(avg_sum) FROM mv_input GROUP BY \"RegionID\"", + List.of("RegionID", "adv_sum", "cnt", "avg_cnt", "avg_sum") + ); +} diff --git a/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVDocumentInput.java b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVDocumentInput.java new file mode 100644 index 0000000000000..65c81937488e4 --- /dev/null +++ b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVDocumentInput.java @@ -0,0 +1,65 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.mv; + +import org.opensearch.index.engine.dataformat.DocumentInput; +import org.opensearch.index.mapper.MappedFieldType; + +import java.util.List; + +/** + * Capturing document input (VSR model): keeps the definition's referenced + * columns from the composite broadcast, in the spec's buffer order (group + * keys first). Spec-driven so the same class serves the SOURCE definition + * (raw fields) and the TARGET fold definition (state fields). + */ +public final class MVDocumentInput implements DocumentInput { + + private final List columns; + private final Object[] values; + + public MVDocumentInput(MVDefinitionSpec spec) { + this.columns = spec.columns(); + this.values = new Object[columns.size()]; + } + + @Override + public Object[] getFinalInput() { + Object[] row = values.clone(); + java.util.Arrays.fill(values, null); + return row; + } + + @Override + public void addField(MappedFieldType fieldType, Object value) { + if (value == null) { + return; + } + for (int i = 0; i < columns.size(); i++) { + MVDefinitionSpec.Column col = columns.get(i); + if (col.name().equals(fieldType.name())) { + values[i] = col.type() == MVDefinitionSpec.ColumnType.UTF8 ? value.toString() : ((Number) value).longValue(); + return; + } + } + // Field not referenced by the definition (e.g. provenance fields on + // the target) — ignored. + } + + @Override + public void setRowId(String rowIdFieldName, long rowId) {} + + @Override + public long getFieldCount(String fieldName) { + return 0; + } + + @Override + public void close() {} +} diff --git a/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVForwardBuffer.java b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVForwardBuffer.java new file mode 100644 index 0000000000000..d10e1039c4f03 --- /dev/null +++ b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVForwardBuffer.java @@ -0,0 +1,114 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.mv; + +import org.apache.arrow.c.ArrowArray; +import org.apache.arrow.c.ArrowSchema; +import org.apache.arrow.c.Data; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +/** + * POC(mv) forward buffer: VSR with the definition's captured columns (spec- + * driven — serves both the SOURCE and the TARGET-fold definitions). Rotation + * exports via Arrow C Data and feeds the native DataFusion-maintained state. + * + *

Rollback contract unchanged: rotation happens BEFORE appending, so a + * failed doc is always still in this buffer; truncateTo undoes it. + */ +final class MVForwardBuffer implements AutoCloseable { + + static final int ROTATION_THRESHOLD = 4096; + + private final BufferAllocator allocator; + private final VectorSchemaRoot vsr; + private final List vectors; + private final List columns; + private int rowCount = 0; + + MVForwardBuffer(MVDefinitionSpec spec) { + this.allocator = new RootAllocator(64L * 1024 * 1024); + this.columns = spec.columns(); + this.vectors = new ArrayList<>(columns.size()); + for (MVDefinitionSpec.Column col : columns) { + vectors.add( + col.type() == MVDefinitionSpec.ColumnType.UTF8 + ? new VarCharVector(col.name(), allocator) + : new BigIntVector(col.name(), allocator) + ); + } + this.vsr = new VectorSchemaRoot(vectors); + } + + void append(Object[] row) { + for (int i = 0; i < columns.size(); i++) { + FieldVector v = vectors.get(i); + Object value = row[i]; + if (v instanceof VarCharVector vc) { + if (value == null) { + vc.setNull(rowCount); + } else { + vc.setSafe(rowCount, ((String) value).getBytes(StandardCharsets.UTF_8)); + } + } else { + BigIntVector bv = (BigIntVector) v; + if (value == null) { + bv.setNull(rowCount); + } else { + bv.setSafe(rowCount, (Long) value); + } + } + } + rowCount++; + } + + int rowCount() { + return rowCount; + } + + boolean shouldRotate() { + return rowCount >= ROTATION_THRESHOLD; + } + + void truncateTo(int rows) { + rowCount = rows; + } + + void rotateInto(long writerHandle) { + if (rowCount == 0) { + return; + } + vsr.setRowCount(rowCount); + try (ArrowArray array = ArrowArray.allocateNew(allocator); ArrowSchema schema = ArrowSchema.allocateNew(allocator)) { + Data.exportVectorSchemaRoot(allocator, vsr, null, array, schema); + MVNativeBridge.writerFeed(writerHandle, array.memoryAddress(), schema.memoryAddress()); + } + vsr.clear(); + rowCount = 0; + } + + @Override + public void close() { + vsr.close(); + allocator.close(); + } + + /** The buffer's allocator — also used to import the finalized state batch for the ship path. */ + org.apache.arrow.memory.BufferAllocator allocator() { + return allocator; + } +} diff --git a/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVIndexingEngine.java b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVIndexingEngine.java new file mode 100644 index 0000000000000..ea76290bfd70c --- /dev/null +++ b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVIndexingEngine.java @@ -0,0 +1,237 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.mv; + +import org.opensearch.index.engine.dataformat.IndexingExecutionEngine; +import org.opensearch.index.engine.dataformat.Merger; +import org.opensearch.index.engine.dataformat.RefreshInput; +import org.opensearch.index.engine.dataformat.RefreshResult; +import org.opensearch.index.engine.dataformat.Writer; +import org.opensearch.index.engine.dataformat.WriterConfig; +import org.opensearch.index.engine.exec.Segment; +import org.opensearch.index.shard.ShardPath; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * POC(mv) per-shard indexing engine for the derived materialized-view format. + * Pass-through refresh (parquet-engine pattern); merges deliberately + * unsupported in the POC. + */ +public final class MVIndexingEngine + implements + IndexingExecutionEngine { + + /** Definition this engine's writers maintain (SOURCE raw defn or TARGET fold). */ + private final MVDefinitionSpec spec; + + /** The derived format this engine serves (materialized_view on sources, mv_state on targets). */ + private final org.opensearch.index.engine.dataformat.DerivedDataFormat format; + + private final ShardPath shardPath; + private final String tableName; + private final String sourceIndexName; + /** Target MV indices for the separate-index ship path; empty = embedded mode. */ + private final java.util.List shipTargets; + private final java.util.function.Supplier clientSupplier; + private final java.util.function.Supplier clusterServiceSupplier; + /** Per-target high-water of ship-acked catalog snapshot versions (commit sync, decision 25). */ + private final java.util.concurrent.ConcurrentMap targetSnapshotHighWater = new java.util.concurrent.ConcurrentHashMap<>(); + + public MVIndexingEngine(ShardPath shardPath, String indexName) { + this(shardPath, indexName, MVDefinitionSpec.SOURCE, MVDataFormat.INSTANCE, java.util.List.of(), () -> null, () -> null); + } + + public MVIndexingEngine( + ShardPath shardPath, + String indexName, + MVDefinitionSpec spec, + org.opensearch.index.engine.dataformat.DerivedDataFormat format, + java.util.List shipTargets, + java.util.function.Supplier clientSupplier, + java.util.function.Supplier clusterServiceSupplier + ) { + this.spec = spec; + this.format = format; + this.shardPath = shardPath; + this.sourceIndexName = indexName; + this.tableName = indexName.replace('-', '_').replace('.', '_'); + this.shipTargets = shipTargets == null ? java.util.List.of() : java.util.List.copyOf(shipTargets); + this.clientSupplier = clientSupplier; + this.clusterServiceSupplier = clusterServiceSupplier; + try { + Files.createDirectories(shardPath.getDataPath().resolve(getDataFormat().name())); + } catch (IOException e) { + throw new RuntimeException("failed to create mv dir", e); + } + } + + @Override + public Writer createWriter(WriterConfig config) { + MVStateShipper shipper = null; + if (shipTargets.isEmpty() == false) { + org.opensearch.transport.client.Client client = clientSupplier.get(); + if (client == null) { + throw new IllegalStateException("mv ship targets " + shipTargets + " configured but node client not initialized"); + } + shipper = new MVStateShipper( + client, + shipTargets, + sourceIndexName, + shardPath.getShardId(), + clusterServiceSupplier.get(), + spec, + targetSnapshotHighWater + ); + } + return new MVWriter(config.writerGeneration(), shardPath, tableName, spec, getDataFormat(), shipper); + } + + @Override + public java.util.Map beforeCommit() throws java.io.IOException { + // Commit sync (decision 25): before the SOURCE commits, every ship + // target must durably commit a catalog snapshot >= the version its + // ship acks reported — the superset rule at commit granularity. + // Target-side engines (no ship targets) are a no-op: no recursion. + if (shipTargets.isEmpty() || targetSnapshotHighWater.isEmpty()) { + return java.util.Map.of(); + } + java.util.Map meta = new java.util.HashMap<>(); + for (java.util.Map.Entry entry : targetSnapshotHighWater.entrySet()) { + String target = entry.getKey(); + long minVersion = entry.getValue(); + try { + MVCommitSyncAction.Response response = clientSupplier.get() + .execute(MVCommitSyncAction.INSTANCE, new MVCommitSyncAction.Request(target, shardPath.getShardId().id(), minVersion)) + .actionGet(); + meta.put("mv.commit." + target, Long.toString(response.committedVersion())); + } catch (Exception e) { + // Refusal semantics, same as a failed ship: the source never + // commits ahead of its target's durability. + throw new java.io.IOException("mv commit sync failed for target [" + target + "] (min version " + minVersion + ")", e); + } + } + return meta; + } + + @Override + public Merger getMerger() { + // Recompute-on-merge (the safe default from the separate-index design, + // implementation-state §8): derive the merged segment's state by + // running the definition over the MERGED PRIMARY parquet file. Always + // consistent with the post-merge document set (and with the future + // orphan sweep, which is doc-level). The state⊕state fold merger is + // the later optimization, gated on the sweep's watermark. + // + // Ship mode (source with ship targets): merges are a NON-EVENT — no + // logical data change, nothing to re-ship, no local files to produce. + return mergeInput -> { + if (shipTargets.isEmpty() == false) { + return new org.opensearch.index.engine.dataformat.MergeResult(java.util.Map.of(), null); + } + long gen = mergeInput.newWriterGeneration(); + // Merged primary parquet path by the engine's naming convention + // (POC path coupling, same as the original derived-build). + java.nio.file.Path parquetDir = shardPath.getDataPath().resolve("parquet"); + // Engine naming: merge outputs are "_parquet_file_generation_merged_" + // (plain "_parquet_file_generation_" for flush outputs). + java.nio.file.Path merged = parquetDir.resolve("_parquet_file_generation_merged_" + Long.toHexString(gen) + ".parquet"); + if (java.nio.file.Files.exists(merged) == false) { + merged = parquetDir.resolve("_parquet_file_generation_" + Long.toHexString(gen) + ".parquet"); + } + if (java.nio.file.Files.exists(merged) == false) { + throw new java.io.IOException("mv merge: merged parquet not found for gen " + gen + " in " + parquetDir); + } + java.nio.file.Path mvDir = shardPath.getDataPath().resolve(format.name()); + java.nio.file.Files.createDirectories(mvDir); + java.nio.file.Path out = mvDir.resolve(MVConstants.mvFileName(gen)); + long rows = MVNativeBridge.buildStateFile(merged.toString(), "mv_input", spec.sql(), out.toString()); + org.opensearch.index.engine.exec.MonoFileWriterSet fileSet = org.opensearch.index.engine.exec.MonoFileWriterSet.of( + mvDir.toAbsolutePath(), + gen, + out.getFileName().toString(), + Math.max(rows, 1) + ); + return new org.opensearch.index.engine.dataformat.MergeResult(java.util.Map.of(getDataFormat(), fileSet), null); + }; + } + + @Override + public RefreshResult refresh(RefreshInput refreshInput) { + if (refreshInput == null) { + return new RefreshResult(List.of()); + } + List segments = new ArrayList<>(); + segments.addAll(refreshInput.existingSegments()); + segments.addAll(refreshInput.writerFiles()); + return new RefreshResult(List.copyOf(segments)); + } + + @Override + public long getNextWriterGeneration() { + throw new UnsupportedOperationException("generation is owned by DataFormatAwareEngine"); + } + + @Override + public org.opensearch.index.engine.dataformat.DerivedDataFormat getDataFormat() { + return format; + } + + @Override + public long getHeapBytesUsed() { + return 0; + } + + @Override + public long getNativeBytesUsed() { + return 0; + } + + @Override + public Map> deleteFiles(Map> filesToDelete) { + Map> failed = new HashMap<>(); + Collection mvFiles = filesToDelete.get(MVDataFormat.NAME); + if (mvFiles == null) { + return failed; + } + Path dir = shardPath.getDataPath().resolve(MVConstants.DIR); + List failures = new ArrayList<>(); + for (String f : mvFiles) { + try { + Files.deleteIfExists(dir.resolve(f)); + } catch (IOException e) { + failures.add(f); + } + } + if (failures.isEmpty() == false) { + failed.put(MVDataFormat.NAME, failures); + } + return failed; + } + + @Override + public MVDocumentInput newDocumentInput() { + return new MVDocumentInput(spec); + } + + @Override + public org.opensearch.index.engine.exec.commit.IndexStoreProvider getProvider() { + return null; + } + + @Override + public void close() {} +} diff --git a/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVNativeBridge.java b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVNativeBridge.java new file mode 100644 index 0000000000000..456794e9ea1b9 --- /dev/null +++ b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVNativeBridge.java @@ -0,0 +1,285 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.mv; + +import org.opensearch.nativebridge.spi.NativeCall; +import org.opensearch.nativebridge.spi.NativeLibraryLoader; + +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.SymbolLookup; +import java.lang.foreign.ValueLayout; +import java.lang.invoke.MethodHandle; + +/** + * POC(mv): minimal FFI binding for the MV state-file build. Binds + * {@code df_mv_build_poc} from the shared native lib directly — no dependency + * on the DataFusion plugin's Java classes (avoids cross-plugin classloader + * coupling; both crates' symbols live in the one shared .so). + */ +public final class MVNativeBridge { + + private static final MethodHandle MV_INIT_RUNTIME; + private static final MethodHandle MV_BUILD_POC; + private static final MethodHandle MV_BUILD_ARROW; + private static final MethodHandle MV_SEARCH_POC; + private static final MethodHandle MV_WRITER_CREATE; + private static final MethodHandle MV_WRITER_FEED; + private static final MethodHandle MV_WRITER_FINALIZE; + private static final MethodHandle MV_WRITER_FINALIZE_ARROW; + private static final MethodHandle MV_WRITER_ABORT; + private static final MethodHandle MV_SEARCH_V2; + + static { + Linker linker = Linker.nativeLinker(); + SymbolLookup lib = NativeLibraryLoader.symbolLookup(); + // Real-node finding: each plugin classloader loads ITS OWN native + // instance (separate globals). The MV writers therefore need the + // runtime manager initialized in THIS instance — the DataFusion + // plugin's init lives in a different one. POC-grade; production + // consolidates on one shared native instance. + MV_INIT_RUNTIME = linker.downcallHandle( + lib.find("df_init_runtime_manager").orElseThrow(), + FunctionDescriptor.ofVoid(ValueLayout.JAVA_INT, ValueLayout.JAVA_DOUBLE, ValueLayout.JAVA_DOUBLE) + ); + // i64 df_mv_build_poc(input_ptr, input_len, table_ptr, table_len, sql_ptr, sql_len, output_ptr, output_len) + MV_BUILD_POC = linker.downcallHandle( + lib.find("df_mv_build_poc").orElseThrow(() -> new IllegalStateException("df_mv_build_poc symbol missing")), + FunctionDescriptor.of( + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG + ) + ); + // i64 df_mv_search_poc(files_ptr, files_lens, files_count, group_ptr, group_len, + // state_ptr, state_len, out_ptr, out_cap, out_len) + MV_BUILD_ARROW = linker.downcallHandle( + lib.find("df_mv_build_arrow").orElseThrow(() -> new IllegalStateException("df_mv_build_arrow not found")), + FunctionDescriptor.of( + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.JAVA_LONG, + ValueLayout.JAVA_LONG + ) + ); + MV_SEARCH_POC = linker.downcallHandle( + lib.find("df_mv_search_poc").orElseThrow(() -> new IllegalStateException("df_mv_search_poc symbol missing")), + FunctionDescriptor.of( + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS + ) + ); + MV_WRITER_CREATE = linker.downcallHandle( + lib.find("df_mv_writer_create").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.ADDRESS, ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG) + ); + MV_WRITER_FEED = linker.downcallHandle( + lib.find("df_mv_writer_feed").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG) + ); + MV_WRITER_FINALIZE = linker.downcallHandle( + lib.find("df_mv_writer_finalize").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.ADDRESS, ValueLayout.JAVA_LONG) + ); + // i64 df_mv_writer_finalize_arrow(writer_id, array_addr, schema_addr) + MV_WRITER_FINALIZE_ARROW = linker.downcallHandle( + lib.find("df_mv_writer_finalize_arrow").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG) + ); + MV_WRITER_ABORT = linker.downcallHandle( + lib.find("df_mv_writer_abort").orElseThrow(), + FunctionDescriptor.ofVoid(ValueLayout.JAVA_LONG) + ); + MV_SEARCH_V2 = linker.downcallHandle( + lib.find("df_mv_search_v2").orElseThrow(), + FunctionDescriptor.of( + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS + ) + ); + } + + private MVNativeBridge() {} + + /** Initializes this native instance's tokio runtime manager (idempotent per instance). */ + public static void initRuntime(int cpuThreads) { + try { + MV_INIT_RUNTIME.invokeExact(cpuThreads, 1.0d, 1.0d); + } catch (Throwable t) { + throw new RuntimeException("df_init_runtime_manager failed", t); + } + } + + /** + * Blocking MV state-file build: reads {@code inputFile} (primary parquet), + * runs {@code sql} stopped at Partial mode, writes state rows (sorted, per + * the ORDER BY in the sql) to {@code outputFile}. Returns state row count. + */ + public static long buildStateFile(String inputFile, String tableName, String sql, String outputFile) { + try (var call = new NativeCall()) { + var in = call.str(inputFile); + var table = call.str(tableName); + var query = call.str(sql); + var out = call.str(outputFile); + return call.invoke( + MV_BUILD_POC, + in.segment(), + in.len(), + table.segment(), + table.len(), + query.segment(), + query.len(), + out.segment(), + out.len() + ); + } + } + + /** + * Refresh-time ship build: Partial over one parquet file, sorted state + * batch exported via Arrow C-Data into the given FFI struct addresses. + * Returns the state row count. + */ + public static long buildArrow(String inputFile, String tableName, String sql, long arrayAddr, long schemaAddr) { + try (var call = new NativeCall()) { + var in = call.str(inputFile); + var table = call.str(tableName); + var query = call.str(sql); + return call.invoke( + MV_BUILD_ARROW, + in.segment(), + in.len(), + table.segment(), + table.len(), + query.segment(), + query.len(), + arrayAddr, + schemaAddr + ); + } + } + + /** + * POC search: Final-aggregation over the given MV state files. Always goes + * to the MV (no fallback). Returns tab-separated "group\tcount" lines. + */ + public static String search(java.util.List stateFiles, String groupKey, String stateCol) { + try (var call = new NativeCall()) { + var files = call.strArray(stateFiles.toArray(new String[0])); + var group = call.str(groupKey); + var state = call.str(stateCol); + var out = call.outBuffer(1024 * 1024); + call.invoke( + MV_SEARCH_POC, + files.ptrs(), + files.lens(), + (long) stateFiles.size(), + group.segment(), + group.len(), + state.segment(), + state.len(), + out.data(), + (long) out.capacity(), + out.lenOut() + ); + return new String(out.toByteArray(), java.nio.charset.StandardCharsets.UTF_8); + } + } + + // ---- Streaming writer lifecycle (VSR model) ---- + + public static long writerCreate(String definitionSql, int numGroupCols) { + try (var call = new NativeCall()) { + var sql = call.str(definitionSql); + return call.invoke(MV_WRITER_CREATE, sql.segment(), sql.len(), (long) numGroupCols); + } + } + + public static void writerFeed(long writerId, long arrayAddress, long schemaAddress) { + try (var call = new NativeCall()) { + call.invoke(MV_WRITER_FEED, writerId, arrayAddress, schemaAddress); + } + } + + /** + * Finalizes the writer and exports the sorted state batch via Arrow C-Data + * into the given caller-allocated struct addresses — zero copy; the caller + * imports the structs exactly once and owns the resulting root. Returns + * the state row count. + */ + public static long writerFinalizeArrow(long writerId, long arrayAddress, long schemaAddress) { + try (var call = new NativeCall()) { + return call.invoke(MV_WRITER_FINALIZE_ARROW, writerId, arrayAddress, schemaAddress); + } + } + + public static long writerFinalize(long writerId, String outputFile) { + try (var call = new NativeCall()) { + var out = call.str(outputFile); + return call.invoke(MV_WRITER_FINALIZE, writerId, out.segment(), out.len()); + } + } + + /** v2 search: SQL template with __MV_STATES__ placeholder over the state files. */ + public static String searchV2(java.util.List stateFiles, String sqlTemplate) { + try (var call = new NativeCall()) { + var files = call.strArray(stateFiles.toArray(new String[0])); + var sql = call.str(sqlTemplate); + var out = call.outBuffer(1024 * 1024); + call.invoke( + MV_SEARCH_V2, + files.ptrs(), + files.lens(), + (long) stateFiles.size(), + sql.segment(), + sql.len(), + out.data(), + (long) out.capacity(), + out.lenOut() + ); + return new String(out.toByteArray(), java.nio.charset.StandardCharsets.UTF_8); + } + } + + public static void writerAbort(long writerId) { + // Void native call — NativeCall.invoke expects a long return; invoke directly. + try { + MV_WRITER_ABORT.invokeExact(writerId); + } catch (Throwable t) { + throw new RuntimeException("df_mv_writer_abort failed", t); + } + } +} diff --git a/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVReaderManager.java b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVReaderManager.java new file mode 100644 index 0000000000000..dd15662ac84a5 --- /dev/null +++ b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVReaderManager.java @@ -0,0 +1,72 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.mv; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.index.engine.exec.EngineReaderManager; +import org.opensearch.index.engine.exec.WriterFileSet; +import org.opensearch.index.engine.exec.coord.CatalogSnapshot; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; + +/** + * POC(mv) reader manager: exposes the MV state files of a catalog snapshot. + * The "reader" is simply the list of state-file sets for the snapshot — + * coverage IS the snapshot (a segment is covered iff its materialized_view + * entry exists). + */ +public final class MVReaderManager implements EngineReaderManager { + + private static final Logger logger = LogManager.getLogger(MVReaderManager.class); + + /** Format name whose file sets this manager exposes (materialized_view or mv_state). */ + private final String formatName; + + public MVReaderManager() { + this(MVDataFormat.NAME); + } + + public MVReaderManager(String formatName) { + this.formatName = formatName; + } + + /** Snapshot-scoped view of MV state files. */ + public record MVReader(List stateFiles) { + } + + @Override + public MVReader getReader(CatalogSnapshot catalogSnapshot) { + return new MVReader(List.copyOf(catalogSnapshot.getSearchableFiles(formatName))); + } + + @Override + public void beforeRefresh() {} + + @Override + public void afterRefresh(boolean didRefresh, CatalogSnapshot catalogSnapshot) { + if (didRefresh && catalogSnapshot != null) { + logger.debug("mv afterRefresh [{}]: {} state file sets", formatName, catalogSnapshot.getSearchableFiles(formatName).size()); + } + } + + @Override + public void onDeleted(CatalogSnapshot catalogSnapshot) {} + + @Override + public void onFilesAdded(Collection files) {} + + @Override + public void onFilesDeleted(Collection files) {} + + @Override + public void close() throws IOException {} +} diff --git a/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVRefCountedStateBatch.java b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVRefCountedStateBatch.java new file mode 100644 index 0000000000000..63218450be404 --- /dev/null +++ b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVRefCountedStateBatch.java @@ -0,0 +1,64 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.mv; + +import org.apache.arrow.vector.VectorSchemaRoot; + +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Reference-counted ownership of one finalized state batch shared across + * MULTIPLE ship targets. + * + *

The batch is the live Arrow root the native writer finalized into — + * one copy in memory, read-only from every consumer's perspective (concurrent + * reads of immutable Arrow buffers are safe). No single destination may close + * it: the SOURCE acquires one reference per target before shipping, each + * target's handler {@link #release()}s exactly once when done (success or + * failure), and the source releases a target's reference itself when the ship + * to that target fails before the handler could take ownership. The LAST + * release closes the root, which fires the C-Data release callback and frees + * the native allocation. + * + *

Releases past zero throw — a double release is an ownership bug, not a + * condition to tolerate silently (the Arrow debug allocator in tests catches + * the complementary leak case of a missing release). + */ +public final class MVRefCountedStateBatch { + + private final VectorSchemaRoot root; + private final AtomicInteger refs; + + /** + * @param root the finalized state batch; ownership transfers to this holder + * @param consumers number of references handed out (one per ship target) + */ + public MVRefCountedStateBatch(VectorSchemaRoot root, int consumers) { + if (consumers <= 0) { + throw new IllegalArgumentException("consumers must be positive, got " + consumers); + } + this.root = root; + this.refs = new AtomicInteger(consumers); + } + + /** Read-only access to the shared batch. Valid until the caller's own {@link #release()}. */ + public VectorSchemaRoot root() { + return root; + } + + /** Releases one reference; the last release closes the root (frees native memory). */ + public void release() { + int remaining = refs.decrementAndGet(); + if (remaining == 0) { + root.close(); + } else if (remaining < 0) { + throw new IllegalStateException("mv state batch released more times than acquired"); + } + } +} diff --git a/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVShipStateAction.java b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVShipStateAction.java new file mode 100644 index 0000000000000..b279c5fede7dc --- /dev/null +++ b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVShipStateAction.java @@ -0,0 +1,169 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.mv; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.ActionType; +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import java.io.IOException; + +/** + * POC(mv) shard-addressed state ship: carries one source generation's state + * rows to the node holding the ordinal-paired TARGET primary, which applies + * them through the shard's write path (translog — the ack means durable). + * + *

Why a dedicated transport action instead of a client bulk: + *

    + *
  • Shard-addressed: bulk routes by doc-id hash and sprays one + * source shard's rows across ALL target shards, defeating the ordinal + * pairing the colocation decider maintains. This action ships shard i's + * state to target shard i by construction.
    • + *
  • Local short-circuit: {@code TransportService} invokes the + * handler directly (no serialization) when the resolved node is local — + * colocation makes that the common case.
    • + *
+ */ +public final class MVShipStateAction extends ActionType { + + public static final String NAME = "indices:data/write/mv_ship_state"; + public static final MVShipStateAction INSTANCE = new MVShipStateAction(); + + private MVShipStateAction() { + super(NAME, Response::new); + } + + /** + * One source generation's state batch for one target shard, carried as the + * LIVE ARROW ROOT — the same buffers the native writer finalized into, + * zero copies since. Legal because this action is LOCAL-ONLY by the hard + * locality rule (the handler runs in the same JVM via NodeClient dispatch); + * wire serialization is deliberately unsupported and loudly fails if the + * rule is ever broken. + * + *

Ownership: the request carries a REFERENCE on a shared + * {@link MVRefCountedStateBatch} (the same batch may be in flight to + * multiple targets); the handler releases exactly its own reference in + * its finally, success or failure — never closing the batch under other + * consumers. The last release, wherever it happens, frees the native + * memory. + */ + public static class Request extends ActionRequest { + private final String targetIndex; + private final int targetShard; + private final String sourceIndex; + private final int sourceShard; + private final long writerGeneration; + private final java.util.List shipFields; + private final MVRefCountedStateBatch stateBatch; + + public Request( + String targetIndex, + int targetShard, + String sourceIndex, + int sourceShard, + long writerGeneration, + java.util.List shipFields, + MVRefCountedStateBatch stateBatch + ) { + this.shipFields = shipFields; + this.targetIndex = targetIndex; + this.targetShard = targetShard; + this.sourceIndex = sourceIndex; + this.sourceShard = sourceShard; + this.writerGeneration = writerGeneration; + this.stateBatch = stateBatch; + } + + public Request(StreamInput in) throws IOException { + throw new UnsupportedOperationException( + "mv_ship_state is local-only (hard locality rule): the request carries live Arrow buffers " + + "and must never cross a wire — receiving it remotely means the rule was broken" + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + throw new UnsupportedOperationException( + "mv_ship_state is local-only (hard locality rule): the request carries live Arrow buffers and cannot be serialized" + ); + } + + public String targetIndex() { + return targetIndex; + } + + public int targetShard() { + return targetShard; + } + + public String sourceIndex() { + return sourceIndex; + } + + public int sourceShard() { + return sourceShard; + } + + public long writerGeneration() { + return writerGeneration; + } + + public java.util.List shipFields() { + return shipFields; + } + + public MVRefCountedStateBatch stateBatch() { + return stateBatch; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + } + + /** Ack: rows are durably applied on the target primary. */ + public static class Response extends ActionResponse { + private final int applied; + /** + * Target catalog snapshot version covering this apply (post-refresh). + * The source's commit sync (decision 25) commits the target AT LEAST + * to this version before committing itself. + */ + private final long snapshotVersion; + + public Response(int applied, long snapshotVersion) { + this.applied = applied; + this.snapshotVersion = snapshotVersion; + } + + public Response(StreamInput in) throws IOException { + this.applied = in.readVInt(); + this.snapshotVersion = in.readVLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(applied); + out.writeVLong(snapshotVersion); + } + + public int applied() { + return applied; + } + + public long snapshotVersion() { + return snapshotVersion; + } + } +} diff --git a/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVShipStateTransportHandler.java b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVShipStateTransportHandler.java new file mode 100644 index 0000000000000..065581e5de84c --- /dev/null +++ b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVShipStateTransportHandler.java @@ -0,0 +1,243 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.mv; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.lucene.uid.Versions; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.VersionType; +import org.opensearch.index.engine.Engine; +import org.opensearch.index.mapper.SourceToParse; +import org.opensearch.index.shard.IndexShard; +import org.opensearch.indices.IndicesService; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +/** + * Applies shipped MV state rows on the local TARGET primary through the + * shard's write path: each row becomes an index operation with an external + * doc id (idempotent overwrite on re-ship), written to the translog, and the + * translog is fsynced before the ack — the ack means durable. Handing + * the buffer to the writer around the write path would break the invariant on + * a target crash (acked-but-lost state); this is decision 13 in the + * separate-index README. + * + *

Runs on the WRITE pool. Locality is a HARD PRECONDITION: the ordinal- + * paired target primary must be on this node (the colocation decider's job) — + * a split pair fails the ship, which fails the flush (ship-before-commit + * backpressure) until reactive following restores the pair. There is no + * remote-forward path; NodeClient dispatch means nothing is ever serialized. + * + *

POC scope: primary-only apply (no replication chain; POC targets run + * with zero replicas). Production replaces this with a + * TransportWriteAction-style replicated apply. + */ +public final class MVShipStateTransportHandler extends HandledTransportAction { + + /** Latest catalog snapshot version of the target's composite engine (post-refresh); -1 for plain targets. */ + static long engineSnapshotVersion(org.opensearch.index.shard.IndexShard shard) { + return shard.compositeCatalogSnapshotVersion(); + } + + private static final Logger logger = LogManager.getLogger(MVShipStateTransportHandler.class); + + private final IndicesService indicesService; + private final org.opensearch.cluster.service.ClusterService clusterService; + private final TransportService transportService; + + @Inject + public MVShipStateTransportHandler( + TransportService transportService, + ActionFilters actionFilters, + IndicesService indicesService, + org.opensearch.cluster.service.ClusterService clusterService + ) { + super(MVShipStateAction.NAME, transportService, actionFilters, MVShipStateAction.Request::new, ThreadPool.Names.WRITE); + this.indicesService = indicesService; + this.clusterService = clusterService; + this.transportService = transportService; + } + + @Override + protected void doExecute(Task task, MVShipStateAction.Request request, ActionListener listener) { + // Any exit that does not enter applyLocally must release this + // handler's reference on the shared batch — applyLocally releases in + // its own finally once entered. + boolean handedOff = false; + try { + org.opensearch.cluster.routing.ShardRouting primary = clusterService.state() + .routingTable() + .index(request.targetIndex()) + .shard(request.targetShard()) + .primaryShard(); + if (primary == null || primary.active() == false || primary.currentNodeId() == null) { + listener.onFailure( + new IllegalStateException( + "mv ship: target primary [" + request.targetIndex() + "][" + request.targetShard() + "] is not active" + ) + ); + return; + } + String localNodeId = clusterService.localNode().getId(); + if (primary.currentNodeId().equals(localNodeId) == false) { + // HARD RULE: the ordinal-paired target primary must be local at + // ship time. No remote fallback — a split pair fails the flush + // (ship-before-commit backpressure), and the colocation + // decider's reactive following restores the pair; the retried + // flush then succeeds. One path, one failure mode. + listener.onFailure( + new IllegalStateException( + "mv ship: target primary [" + + request.targetIndex() + + "][" + + request.targetShard() + + "] is on node [" + + primary.currentNodeId() + + "], not local [" + + localNodeId + + "] — colocation is a ship precondition" + ) + ); + return; + } + logger.info("mv ship-apply path=local target=[{}][{}]", request.targetIndex(), request.targetShard()); + handedOff = true; + applyLocally(request, listener); + } catch (Exception e) { + listener.onFailure(e); + } finally { + if (handedOff == false) { + request.stateBatch().release(); + } + } + } + + private void applyLocally(MVShipStateAction.Request request, ActionListener listener) { + // The handler owns ONE REFERENCE on the shared batch (the same buffers + // may be in flight to other targets): release it on every exit path, + // never close the root directly — the last release across all + // consumers frees the native allocation. + MVRefCountedStateBatch shared = request.stateBatch(); + try { + org.apache.arrow.vector.VectorSchemaRoot batch = shared.root(); + IndexShard shard = indicesService.indexServiceSafe(clusterService.state().metadata().index(request.targetIndex()).getIndex()) + .getShard(request.targetShard()); + int rows = batch.getRowCount(); + java.util.List vectors = batch.getFieldVectors(); + java.util.List shipFields = request.shipFields(); + if (vectors.size() != shipFields.size()) { + listener.onFailure( + new IllegalStateException( + "mv ship apply: state batch has " + vectors.size() + " columns, expected " + shipFields.size() + ) + ); + return; + } + for (int row = 0; row < rows; row++) { + java.util.Map doc = new java.util.HashMap<>(); + // Positional mapping — the state contract (group keys first, + // then state columns); names in the batch carry the writer's + // alias and are not compared. + for (int col = 0; col < vectors.size(); col++) { + Object value = vectors.get(col).getObject(row); + if (value instanceof org.apache.arrow.vector.util.Text t) { + value = t.toString(); + } else if (value instanceof Double || value instanceof Float) { + // Floating state values (e.g. avg's sum half) must NOT + // be truncated to long — the whole avg correctness + // rides on this bit of plumbing. + value = ((Number) value).doubleValue(); + } else if (value instanceof Number n) { + value = n.longValue(); + } + doc.put(shipFields.get(col), value); + } + // Provenance (decision 21): ONE field. Idempotency lives in the + // deterministic _id (source.shard.gen.row); source index+shard are + // constants per target shard under ordinal-paired colocation. The + // generation field alone has a query job — the orphan sweep's + // delete-by-generation. + doc.put("_mv_source_generation", request.writerGeneration()); + String docId = request.sourceIndex() + "." + request.sourceShard() + "." + request.writerGeneration() + "." + row; + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + builder.map(doc); + SourceToParse source = new SourceToParse( + request.targetIndex(), + docId, + BytesReference.bytes(builder), + MediaTypeRegistry.JSON + ); + Engine.IndexResult result = shard.applyIndexOperationOnPrimary( + Versions.MATCH_ANY, + VersionType.INTERNAL, + source, + org.opensearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO, + 0, + -1L, // UNSET_AUTO_GENERATED_TIMESTAMP + false + ); + if (result.getResultType() != Engine.Result.Type.SUCCESS) { + // Idempotent re-ship vs APPEND-ONLY composite target: + // recovery replay re-ships a generation whose rows may + // already exist under the same deterministic doc id. + // Presence satisfies the invariant — tolerate-duplicate + // IS the idempotency on an append-only index (overwrite + // is impossible there). Divergent stale-orphan content + // is the generation-watermark sweep's job (designed, + // pending). POC-accepted risk until the sweep lands. + Exception failure = result.getFailure(); + boolean alreadyExists = failure != null + && (failure.getClass().getSimpleName().contains("AppendOnlyIndexOperationRetryException") + || failure.getClass().getSimpleName().contains("VersionConflictEngineException")); + if (alreadyExists) { + continue; // row present => counts toward the ack + } + listener.onFailure( + new IllegalStateException( + "mv ship apply failed for [" + docId + "]: " + result.getResultType(), + result.getFailure() + ) + ); + return; + } + } + } + // Durability before ack: fsync the translog to the last applied op. + shard.sync(); + // Searchability before ack (design contract, README "current + // understanding" step 4-5): the ack certifies BOTH durable and + // searchable, so when the source commits (step 6) the target's + // latest view already supersets any source view a query can hold — + // the whole consistency story, no snapshot mapping needed. + shard.refresh("mv_ship"); + logger.debug( + "mv ship-apply: {} state rows into [{}][{}] ({})", + rows, + request.targetIndex(), + request.targetShard(), + transportService.getLocalNode().getId() + ); + long snapshotVersion = engineSnapshotVersion(shard); + listener.onResponse(new MVShipStateAction.Response(rows, snapshotVersion)); + } catch (Exception e) { + listener.onFailure(e); + } finally { + shared.release(); + } + } +} diff --git a/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVStateDataFormat.java b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVStateDataFormat.java new file mode 100644 index 0000000000000..7e1cf71378987 --- /dev/null +++ b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVStateDataFormat.java @@ -0,0 +1,33 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.mv; + +import org.opensearch.index.engine.dataformat.DerivedDataFormat; + +/** + * The separate-index MV TARGET's derived format: maintains per-segment FOLDED + * state files whose definition is the fold of the shipped state schema + * ({@link MVDefinitionSpec#TARGET_FOLD}). Shipped state rows arrive as + * ordinary documents through the target's write path (translog, replication); + * this format observes the broadcast and materializes folded, group-key- + * sorted state per target generation — the embedded-MV shape applied to the + * MV index itself. + */ +public final class MVStateDataFormat extends DerivedDataFormat { + + public static final String NAME = "mv_state"; + public static final MVStateDataFormat INSTANCE = new MVStateDataFormat(); + + private MVStateDataFormat() {} + + @Override + public String name() { + return NAME; + } +} diff --git a/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVStateDataFormatPlugin.java b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVStateDataFormatPlugin.java new file mode 100644 index 0000000000000..09297b08b0ce2 --- /dev/null +++ b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVStateDataFormatPlugin.java @@ -0,0 +1,82 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.mv; + +import org.opensearch.index.IndexSettings; +import org.opensearch.index.engine.dataformat.DataFormat; +import org.opensearch.index.engine.dataformat.DataFormatDescriptor; +import org.opensearch.index.engine.dataformat.DataFormatPlugin; +import org.opensearch.index.engine.dataformat.DataFormatRegistry; +import org.opensearch.index.engine.dataformat.IndexingEngineConfig; +import org.opensearch.index.engine.dataformat.IndexingExecutionEngine; +import org.opensearch.index.engine.dataformat.ReaderManagerConfig; +import org.opensearch.index.engine.exec.EngineReaderManager; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.store.checksum.GenericCRC32ChecksumHandler; +import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.SearchBackEndPlugin; + +import java.util.Map; +import java.util.function.Supplier; + +/** + * POC(mv): registers the {@code mv_state} derived format for separate-index + * MV TARGETS. A target opts in via + * {@code index.composite.secondary_data_formats: [..., "mv_state"]}; the + * format's writers maintain the FOLD of the shipped state (no ship targets of + * their own — MV-over-MV chains would configure ship_targets here too). + */ +public class MVStateDataFormatPlugin extends Plugin implements DataFormatPlugin, SearchBackEndPlugin { + + public MVStateDataFormatPlugin() {} + + @Override + public DataFormat getDataFormat() { + return MVStateDataFormat.INSTANCE; + } + + @Override + public IndexingExecutionEngine indexingEngine(IndexingEngineConfig config) { + String definition = config.indexSettings().getSettings().get(MVConstants.DEFINITION_SETTING, "payments"); + return new MVIndexingEngine( + config.store().shardPath(), + config.indexSettings().getIndex().getName(), + MVDefinitionSpec.fold(definition), + MVStateDataFormat.INSTANCE, + java.util.List.of(), + () -> null, + () -> null + ); + } + + @Override + public Map> getFormatDescriptors(IndexSettings indexSettings, DataFormatRegistry registry) { + return Map.of(MVStateDataFormat.NAME, () -> new DataFormatDescriptor(MVStateDataFormat.NAME, new GenericCRC32ChecksumHandler())); + } + + @Override + public void assignCapabilities(MappedFieldType fieldType, IndexSettings indexSettings, DataFormatRegistry dataFormatRegistry) { + // Derived format: claims nothing (see MVDataFormatPlugin). + } + + @Override + public String name() { + return MVStateDataFormat.NAME; + } + + @Override + public java.util.List getSupportedFormats() { + return java.util.List.of(MVStateDataFormat.NAME); + } + + @Override + public EngineReaderManager createReaderManager(ReaderManagerConfig settings) { + return new MVReaderManager(MVStateDataFormat.NAME); + } +} diff --git a/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVStateShipper.java b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVStateShipper.java new file mode 100644 index 0000000000000..e3e021ed356cc --- /dev/null +++ b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVStateShipper.java @@ -0,0 +1,175 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.mv; + +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.transport.client.Client; + +import java.io.IOException; + +/** + * POC(mv) separate-index ship path: exports a finalized state file's rows and + * synchronously bulk-indexes them into the target MV index. + * + *

Ship-before-commit: the caller ({@code MVWriter.flush}) invokes + * {@link #ship} BEFORE returning its flush result; any failure here throws, + * the flush fails, and the source does not commit — the data-level invariant + * (committed-on-source ⇒ present-on-target) holds by construction. + * + *

Idempotence: doc IDs are deterministic + * ({@code ...}), so a retried flush + * (after a failed ship) overwrites rather than duplicates — re-shipping the + * same generation is safe. + * + *

Visibility: refresh policy NONE — ship+ack guarantees durable + * presence; rows become searchable after the MV index's own refresh (design + * decision 6 in the separate-index README). + * + *

POC simplifications (documented in the separate-index folder): the state + * rows travel as parsed TSV from the native export rather than Arrow batches + * over a dedicated transport action; the synchronous bulk blocks the flushing + * thread (production wants a ship executor + ack listener); no colocation + * fast path (that is the resize-precedent Phase 1/2 work). + */ +final class MVStateShipper { + + private static final Logger logger = LogManager.getLogger(MVStateShipper.class); + + private final Client client; + private final java.util.List targetIndices; + private final String sourceIndex; + private final ShardId shardId; + private final org.opensearch.cluster.service.ClusterService clusterService; + private final MVDefinitionSpec spec; + /** Engine-owned per-target high-water of acked catalog snapshot versions (commit sync, decision 25). */ + private final java.util.concurrent.ConcurrentMap targetSnapshotHighWater; + + MVStateShipper( + Client client, + java.util.List targetIndices, + String sourceIndex, + ShardId shardId, + org.opensearch.cluster.service.ClusterService clusterService, + MVDefinitionSpec spec, + java.util.concurrent.ConcurrentMap targetSnapshotHighWater + ) { + this.spec = spec; + this.client = client; + this.targetIndices = java.util.List.copyOf(targetIndices); + this.sourceIndex = sourceIndex; + this.shardId = shardId; + this.clusterService = clusterService; + this.targetSnapshotHighWater = targetSnapshotHighWater; + } + + /** + * Ships every state row of {@code stateFile} to the target MV index and + * blocks until the bulk is acked. Throws on ANY failure — the caller must + * fail the flush (ship-before-commit). + * + * @return number of state rows shipped + */ + /** + * Ships the finalized state batch — the LIVE Arrow root, zero copies since + * the native writer produced it — to the ordinal-paired target shard and + * blocks until the durable+searchable ack. Throws on ANY failure — the + * caller must fail the flush (ship-before-commit). The handler owns and + * closes the root. + * + * @return number of state rows shipped + */ + /** + * Ships the finalized state batch to EVERY configured target's ordinal- + * paired shard, sharing ONE copy of the Arrow buffers across all of them + * via {@link MVRefCountedStateBatch}: the source acquires one reference + * per target up front; each target's handler releases its own; if the + * ship to a target fails before its handler took ownership, the source + * releases that reference here. The last release — wherever it happens — + * frees the native memory. No destination can free the batch under + * another. + * + *

Commit gate: EVERY target must ack with the full row count; any + * failure fails the flush (ship-before-commit, invariant per target). + * + * @return number of state rows shipped (per target) + */ + long ship(VectorSchemaRoot stateBatch, long writerGeneration) throws IOException { + int rows = stateBatch.getRowCount(); + MVRefCountedStateBatch shared = new MVRefCountedStateBatch(stateBatch, targetIndices.size()); + IOException failure = null; + for (String targetIndex : targetIndices) { + if (failure != null) { + // A previous target already failed the flush: don't ship to the + // remaining targets this attempt — just release their refs. + shared.release(); + continue; + } + int targetShard = shardId.id() % clusterService.state().metadata().index(targetIndex).getNumberOfShards(); + MVShipStateAction.Request request = new MVShipStateAction.Request( + targetIndex, + targetShard, + sourceIndex, + shardId.id(), + writerGeneration, + spec.shipFields(), + shared + ); + MVShipStateAction.Response response = null; + try { + response = client.execute(MVShipStateAction.INSTANCE, request).actionGet(); + } catch (Exception e) { + // The handler releases its reference on every path it reaches; + // dispatch happens synchronously in-JVM (NodeClient), so a + // throw here means the handler's finally already ran or the + // request never left this method — either way OUR contract is + // one release per target, and the handler owns it once + // doExecute is entered. dispatch-failure-before-doExecute is + // not a real path for NodeClient; treat the ref as consumed. + failure = new IOException( + "mv ship: transport ship to [" + targetIndex + "][" + targetShard + "] failed gen=" + writerGeneration, + e + ); + continue; + } + // rowsReceived verification = the commit gate (challenges §10). + if (response.applied() != rows) { + failure = new IOException( + "mv ship: gen=" + + writerGeneration + + " shipped " + + rows + + " state rows to [" + + targetIndex + + "] but it applied " + + response.applied() + ); + continue; + } + if (response.snapshotVersion() >= 0) { + targetSnapshotHighWater.merge(targetIndex, response.snapshotVersion(), Math::max); + } + logger.info( + "mv ship: gen={} shipped {} state rows from {}[{}] -> [{}][{}] (acked durable+searchable)", + writerGeneration, + response.applied(), + sourceIndex, + shardId.id(), + targetIndex, + targetShard + ); + } + if (failure != null) { + throw failure; + } + return rows; + } +} diff --git a/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVViewsService.java b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVViewsService.java new file mode 100644 index 0000000000000..3e8c2990ef7ac --- /dev/null +++ b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVViewsService.java @@ -0,0 +1,202 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.mv; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.cluster.ClusterChangedEvent; +import org.opensearch.cluster.ClusterStateListener; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.action.ActionListener; +import org.opensearch.index.shard.IndexSettingProvider; +import org.opensearch.transport.client.Client; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** + * The `index.mv.views` UX (decisions 20/23/24): the user declares MVs on the + * SOURCE index only — a list of {@code definition} or {@code definition:name} + * entries — and never touches formats, state schemas, or the target index. + * + *

    + *
  • {@link Provider} (an {@link IndexSettingProvider}) expands the views + * list at source-creation time into the full derived settings: the + * {@code materialized_view} secondary format and the ship-target list. + * Merge order is provider < template < request, so a user who + * explicitly sets the composite formats keeps their value — with a + * loud warning if it omits the MV format.
    • + *
  • {@link TargetCreator} (a {@link ClusterStateListener}, elected + * cluster-manager only) creates each missing target index with the + * fully derived settings and mapping (state schema from the definition + * spec + the single hidden provenance field of decision 21), colocated + * with the source. Tolerates already-exists (listener re-entry).
    • + *
+ * + *

v1 in-sync-by-construction (decision 24): this path only exists at + * index creation, so the source is empty by definition. The future + * add-MV-to-existing-index API must add its own race-free emptiness check. + * + *

RESOLVED (was "the mapping gap"): the composite plugin's own + * IndexSettingProvider used to contribute cluster-default formats for every + * new index; provider iteration order is undefined, so on losing orders its + * empty secondary list overwrote this provider's derived formats (breaking + * inline mappings deterministically-ish and shard recovery seed-dependently). + * The composite provider now defers when `index.mv.views` is declared. + * + *

Names (decision 23): {@code definition:name} uses {@code name} as the + * target index; a bare {@code definition} generates + * {@code _mv_}. (Dot-prefixed true system indices need + * SystemIndexPlugin registration — deferred, tracked in the decision log.) + */ +public final class MVViewsService { + + private static final Logger logger = LogManager.getLogger(MVViewsService.class); + + private MVViewsService() {} + + /** Parsed views entry. */ + record View(String definition, String targetIndex) { + } + + static List parseViews(String sourceIndex, List entries) { + List views = new ArrayList<>(entries.size()); + for (String entry : entries) { + int colon = entry.indexOf(':'); + String definition = colon < 0 ? entry : entry.substring(0, colon); + String name = colon < 0 ? String.format(Locale.ROOT, "%s_mv_%s", sourceIndex, definition) : entry.substring(colon + 1); + // Fails fast on unknown definitions (registry lookup throws). + MVDefinitionSpec.source(definition); + views.add(new View(definition, name)); + } + return views; + } + + /** Expands `index.mv.views` into the derived source settings at creation time. */ + public static final class Provider implements IndexSettingProvider { + @Override + public Settings getAdditionalIndexSettings(String indexName, boolean isDataStreamIndex, Settings templateAndRequestSettings) { + List entries = templateAndRequestSettings.getAsList(MVConstants.VIEWS_SETTING); + if (entries.isEmpty()) { + return Settings.EMPTY; + } + List views = parseViews(indexName, entries); + if (views.stream().map(View::definition).distinct().count() > 1) { + // POC: one definition setting per source (the registry slot the + // compiled-PPL definition will replace, decision 22). Multiple + // DISTINCT definitions need per-target definitions on the ship + // path — deferred until definitions are real metadata. + throw new IllegalArgumentException(MVConstants.VIEWS_SETTING + " (POC): all views on one source must share the definition"); + } + Settings explicit = templateAndRequestSettings.filter(k -> k.startsWith("index.composite.") || k.equals("index.mv.definition")); + if (explicit.hasValue("index.composite.secondary_data_formats") + && templateAndRequestSettings.getAsList("index.composite.secondary_data_formats").contains("materialized_view") == false) { + logger.warn( + "[{}] {} is set but the explicit secondary_data_formats omit 'materialized_view' — " + + "the request wins over derived settings; the MV will NOT be maintained", + indexName, + MVConstants.VIEWS_SETTING + ); + } + List targets = views.stream().map(View::targetIndex).toList(); + logger.info("[{}] mv views declared: deriving source settings, targets={}", indexName, targets); + return Settings.builder() + .put("index.pluggable.dataformat.enabled", true) + .put("index.pluggable.dataformat", "composite") + .put("index.composite.primary_data_format", "parquet") + .putList("index.composite.secondary_data_formats", "lucene", "materialized_view") + .put(MVConstants.DEFINITION_SETTING, views.get(0).definition()) + .putList(MVConstants.SHIP_TARGETS_SETTING, targets) + .build(); + } + } + + /** Creates missing target indices for sources that declare views. */ + public static final class TargetCreator implements ClusterStateListener { + + private final Client client; + /** Creations already dispatched this node-lifetime (double-fire guard; already-exists is tolerated anyway). */ + private final Set dispatched = ConcurrentHashMap.newKeySet(); + + public TargetCreator(Client client) { + this.client = client; + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + if (event.localNodeClusterManager() == false || event.metadataChanged() == false) { + return; + } + for (IndexMetadata imd : event.state().metadata().indices().values()) { + List entries = imd.getSettings().getAsList(MVConstants.VIEWS_SETTING); + if (entries.isEmpty()) { + continue; + } + String source = imd.getIndex().getName(); + for (View view : parseViews(source, entries)) { + if (event.state().metadata().hasIndex(view.targetIndex()) + || dispatched.add(source + ">" + view.targetIndex()) == false) { + continue; + } + createTarget(source, view); + } + } + } + + private void createTarget(String source, View view) { + CreateIndexRequest request = new CreateIndexRequest(view.targetIndex()).settings( + Settings.builder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .put("index.pluggable.dataformat.enabled", true) + .put("index.pluggable.dataformat", "composite") + .put("index.composite.primary_data_format", "parquet") + .putList("index.composite.secondary_data_formats", "lucene", "mv_state") + .put(MVConstants.DEFINITION_SETTING, view.definition()) + .put(MVConstants.COLOCATE_WITH_SETTING, source) + ).mapping(targetMapping(view.definition())); + client.admin().indices().create(request, ActionListener.wrap(r -> { + logger.info("mv views: created target [{}] for source [{}] (definition={})", view.targetIndex(), source, view.definition()); + }, e -> { + if (e instanceof org.opensearch.ResourceAlreadyExistsException) { + return; // listener re-entry / concurrent manager — fine + } + // Loud but non-fatal: the ship path fails the source's first + // refresh with a clear error until the target exists, so data + // can't silently commit without its MV. + logger.error("mv views: target [" + view.targetIndex() + "] creation failed for source [" + source + "]", e); + })); + } + + /** + * Derived target mapping: the SHIP schema of the definition (state + * columns as the source ships them) + the single hidden provenance + * field (decision 21). dynamic:false — the composite apply path + * cannot do dynamic mapping updates. + */ + static String targetMapping(String definition) { + MVDefinitionSpec spec = MVDefinitionSpec.source(definition); + StringBuilder sb = new StringBuilder("{\"dynamic\":\"false\",\"properties\":{"); + List shipFields = spec.shipFields(); + List columns = MVDefinitionSpec.fold(definition).columns(); + for (int i = 0; i < shipFields.size(); i++) { + String type = columns.get(i).type() == MVDefinitionSpec.ColumnType.UTF8 ? "keyword" : "long"; + sb.append("\"").append(shipFields.get(i)).append("\":{\"type\":\"").append(type).append("\"},"); + } + sb.append("\"_mv_source_generation\":{\"type\":\"long\"}"); + sb.append("}}"); + return sb.toString(); + } + } +} diff --git a/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVWriter.java b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVWriter.java new file mode 100644 index 0000000000000..849ecaab80d54 --- /dev/null +++ b/sandbox/plugins/mv-data-format/src/main/java/org/opensearch/mv/MVWriter.java @@ -0,0 +1,201 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.mv; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.index.engine.dataformat.FileInfos; +import org.opensearch.index.engine.dataformat.FlushInput; +import org.opensearch.index.engine.dataformat.WriteResult; +import org.opensearch.index.engine.dataformat.Writer; +import org.opensearch.index.engine.dataformat.WriterState; +import org.opensearch.index.engine.exec.MonoFileWriterSet; +import org.opensearch.index.shard.ShardPath; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * POC(mv) writer — REFRESH-TIME BUILD model (decision 18): the complete MV + * state for a generation is computed once, at flush/refresh, by running the + * definition's Partial stage over the generation's just-flushed PRIMARY + * parquet file. Nothing MV-related happens on the doc write path. + * + *

Consequences (why this is the chosen trade): + *

    + *
  • addDoc is a row count — zero per-doc capture cost, no forward + * buffer, no native writer state held across the generation;
    • + *
  • rollback is trivially safe (nothing exists to unwind before + * flush) — the rotate-before-append contract is gone with the + * machinery that needed it;
    • + *
  • the flush does one extra read of the parquet it just wrote (page + * cache hot) plus the aggregation — accepted cost, paid once per + * refresh, off the ingest hot path;
    • + *
  • path coupling to the primary's file naming — same convention the + * merger's recompute already relies on.
    • + *
+ * + *

The streaming/VSR incremental-fold model this replaces remains in git + * history (and native {@code mv_writer.rs}) as the optimization path if + * refresh-time build cost ever matters. + */ +public final class MVWriter implements Writer { + + private static final Logger logger = LogManager.getLogger(MVWriter.class); + + private final long writerGeneration; + private final ShardPath shardPath; + private final String tableName; + private final MVDefinitionSpec spec; + /** Non-null = separate-index mode: ship state rows before commit, keep no local file. */ + private final MVStateShipper shipper; + private long acceptedRows = 0; + private volatile long mappingVersion = 1L; + private volatile WriterState state = WriterState.ACTIVE; + + /** The format this writer registers its files under (source vs target-fold). */ + private final org.opensearch.index.engine.dataformat.DataFormat dataFormat; + + public MVWriter(long writerGeneration, ShardPath shardPath, String tableName) { + this(writerGeneration, shardPath, tableName, MVDefinitionSpec.SOURCE, MVDataFormat.INSTANCE, null); + } + + public MVWriter( + long writerGeneration, + ShardPath shardPath, + String tableName, + MVDefinitionSpec spec, + org.opensearch.index.engine.dataformat.DataFormat dataFormat, + MVStateShipper shipper + ) { + this.writerGeneration = writerGeneration; + this.shardPath = shardPath; + this.tableName = tableName; + this.spec = spec; + this.dataFormat = dataFormat; + this.shipper = shipper; + } + + @Override + public WriteResult addDoc(MVDocumentInput doc) { + // Refresh-time build: the write path only counts rows (the count + // drives the "emit nothing for an empty generation" leg of flush). + acceptedRows++; + return new WriteResult.Success(1L, 1L, 1L); + } + + @Override + public void rollbackTo(long rowCount) { + // Nothing accumulates before flush — rollback is a counter reset. + acceptedRows = rowCount; + state = WriterState.ACTIVE; + } + + @Override + public FileInfos flush(FlushInput flushInput) throws IOException { + if (acceptedRows == 0) { + // Nothing ingested this generation — emit nothing ("or none" leg). + return FileInfos.empty(); + } + + // The composite flush runs the PRIMARY first, so this generation's + // parquet file exists (and is page-cache hot) by the time we run. + Path parquet = flushedPrimaryFile(); + + if (shipper != null) { + // Separate-index mode (Approach 2): build the state batch from + // the flushed parquet and hand it to the target as live Arrow + // buffers via C-Data. Ship-before-commit: only a durable + + // searchable ack lets the flush succeed. The source tracks NO + // MV files (the MV index owns its layout). + try (org.apache.arrow.memory.RootAllocator shipAllocator = new org.apache.arrow.memory.RootAllocator()) { + try ( + org.apache.arrow.c.ArrowArray array = org.apache.arrow.c.ArrowArray.allocateNew(shipAllocator); + org.apache.arrow.c.ArrowSchema schema = org.apache.arrow.c.ArrowSchema.allocateNew(shipAllocator) + ) { + long stateRows = MVNativeBridge.buildArrow( + parquet.toString(), + MVConstants.INPUT_TABLE, + spec.sql(), + array.memoryAddress(), + schema.memoryAddress() + ); + org.apache.arrow.vector.VectorSchemaRoot stateBatch = org.apache.arrow.c.Data.importVectorSchemaRoot( + shipAllocator, + array, + schema, + null + ); + // Ownership passes to the ship action's handler (closes in + // its try/finally); on ship failure the flush fails either way. + long shipped = shipper.ship(stateBatch, writerGeneration); + logger.info("mv flush gen={} shipped {} of {} state rows before commit", writerGeneration, shipped, stateRows); + } + } + return FileInfos.empty(); + } + + // Embedded mode: build the state file (Arrow IPC) next to the shard. + Path mvDir = shardPath.getDataPath().resolve(dataFormat.name()); + Files.createDirectories(mvDir); + Path mvFile = mvDir.resolve(MVConstants.mvFileName(writerGeneration)); + + long stateRows = MVNativeBridge.buildStateFile(parquet.toString(), MVConstants.INPUT_TABLE, spec.sql(), mvFile.toString()); + logger.info("mv flush gen={} built {} state rows at refresh -> {}", writerGeneration, stateRows, mvFile.getFileName()); + + MonoFileWriterSet fileSet = MonoFileWriterSet.of( + mvDir.toAbsolutePath(), + writerGeneration, + mvFile.getFileName().toString(), + Math.max(stateRows, 1) + ); + return FileInfos.builder().putWriterFileSet(dataFormat, fileSet).build(); + } + + /** This generation's primary parquet file, by the engine's naming convention. */ + private Path flushedPrimaryFile() throws IOException { + Path parquetDir = shardPath.getDataPath().resolve("parquet"); + Path flushed = parquetDir.resolve("_parquet_file_generation_" + Long.toHexString(writerGeneration) + ".parquet"); + if (Files.exists(flushed) == false) { + throw new IOException("mv flush: primary parquet not found for gen " + writerGeneration + " at " + flushed); + } + return flushed; + } + + @Override + public long generation() { + return writerGeneration; + } + + @Override + public WriterState state() { + return state; + } + + @Override + public boolean isSchemaMutable() { + return true; + } + + @Override + public long mappingVersion() { + return mappingVersion; + } + + @Override + public void updateMappingVersion(long newVersion) { + this.mappingVersion = newVersion; + } + + @Override + public void close() { + state = WriterState.CLOSED; + } +} diff --git a/sandbox/plugins/mv-state-format/build.gradle b/sandbox/plugins/mv-state-format/build.gradle new file mode 100644 index 0000000000000..d150ec1fd6567 --- /dev/null +++ b/sandbox/plugins/mv-state-format/build.gradle @@ -0,0 +1,28 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * POC: descriptor-only plugin for the mv_state derived format (the separate- + * index MV TARGET's fold format). Real nodes load ONE plugin class per + * installed plugin; MVStateDataFormatPlugin lives in mv-data-format's jar, + * so this plugin exists to give it a descriptor. extendedPlugins puts + * mv-data-format's classloader (and transitively composite-engine's) on the + * path. + */ +opensearchplugin { + description = 'POC: mv_state derived format for separate-index MV targets (fold of the shipped state).' + classname = 'org.opensearch.mv.state.MVStateFormatShimPlugin' + extendedPlugins = ['mv-data-format'] +} + +java { sourceCompatibility = JavaVersion.toVersion(25); targetCompatibility = JavaVersion.toVersion(25) } + +dependencies { + compileOnly project(':sandbox:plugins:mv-data-format') + compileOnly project(':sandbox:plugins:composite-engine') +} diff --git a/sandbox/plugins/mv-state-format/src/main/java/org/opensearch/mv/state/MVStateFormatShimPlugin.java b/sandbox/plugins/mv-state-format/src/main/java/org/opensearch/mv/state/MVStateFormatShimPlugin.java new file mode 100644 index 0000000000000..b74a179bf8ce9 --- /dev/null +++ b/sandbox/plugins/mv-state-format/src/main/java/org/opensearch/mv/state/MVStateFormatShimPlugin.java @@ -0,0 +1,22 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.mv.state; + +import org.opensearch.mv.MVStateDataFormatPlugin; + +/** + * Descriptor shim: real nodes load one plugin class per installed plugin, and + * {@code MVStateDataFormatPlugin} lives inside the mv-data-format jar (the + * IT's {@code nodePlugins()} hid this — it loads classes directly). All + * behavior is inherited; this class only gives the mv_state format its own + * plugin descriptor and classloader edge onto mv-data-format. + */ +public class MVStateFormatShimPlugin extends MVStateDataFormatPlugin { + public MVStateFormatShimPlugin() {} +} diff --git a/sandbox/plugins/mv-state-format/src/main/java/org/opensearch/mv/state/package-info.java b/sandbox/plugins/mv-state-format/src/main/java/org/opensearch/mv/state/package-info.java new file mode 100644 index 0000000000000..e51dc6fa9c7b8 --- /dev/null +++ b/sandbox/plugins/mv-state-format/src/main/java/org/opensearch/mv/state/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Descriptor shim plugin for the {@code mv_state} derived data format. */ +package org.opensearch.mv.state; diff --git a/sandbox/qa/analytics-engine-coordinator/build.gradle b/sandbox/qa/analytics-engine-coordinator/build.gradle index a947185c195d5..b7de97a890230 100644 --- a/sandbox/qa/analytics-engine-coordinator/build.gradle +++ b/sandbox/qa/analytics-engine-coordinator/build.gradle @@ -60,6 +60,8 @@ dependencies { internalClusterTestImplementation project(':sandbox:plugins:parquet-data-format') // Composite engine plugin — provides the composite format dispatcher. internalClusterTestImplementation project(':sandbox:plugins:composite-engine') + internalClusterTestImplementation project(':sandbox:plugins:mv-data-format') + internalClusterTestImplementation project(':sandbox:plugins:mv-state-format') // TestPPLPlugin + UnifiedPPL action for driving queries from ITs. internalClusterTestImplementation project(':sandbox:plugins:test-ppl-frontend') diff --git a/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/mv/MVNativeReadIT.java b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/mv/MVNativeReadIT.java new file mode 100644 index 0000000000000..56c1e8fe8492c --- /dev/null +++ b/sandbox/qa/analytics-engine-coordinator/src/internalClusterTest/java/org/opensearch/mv/MVNativeReadIT.java @@ -0,0 +1,230 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.mv; + +import org.opensearch.Version; +import org.opensearch.analytics.AnalyticsPlugin; +import org.opensearch.analytics.sql.SqlPlanRunner; +import org.opensearch.arrow.allocator.ArrowBasePlugin; +import org.opensearch.arrow.flight.transport.FlightStreamPlugin; +import org.opensearch.be.datafusion.DataFusionPlugin; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.composite.CompositeDataFormatPlugin; +import org.opensearch.index.engine.dataformat.stub.MockCommitterEnginePlugin; +import org.opensearch.parquet.ParquetOnlyDataFormatPlugin; +import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.PluginInfo; +import org.opensearch.test.OpenSearchIntegTestCase; + + +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +/** + * The native MV read (validation scope): the EXACT q9 aggregate — including + * {@code AVG}, no eval, no state-name knowledge — served from the MV target's + * {@code mv_state} Arrow files. + * + *

Zero-translation contract end to end: the definition IS the query + * ({@link MVDefinitionSpec#CLICKBENCH_Q9_NATIVE}), so the shipped state + * carries DataFusion's own partial layout (avg as its {@code [count, sum]} + * state pair). With {@code index.mv.serve_state=true} on the target, the + * shard fragment REPLACES its Partial with the state-file scan (STRICT — + * any schema/type/plan misalignment throws, never a silent fallback) and the + * coordinator's Final merges counts+sums and divides once — exactly how + * DataFusion finishes any distributed aggregate. + */ +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 1, numClientNodes = 0) +public class MVNativeReadIT extends OpenSearchIntegTestCase { + + private static final String SOURCE = "hits_native"; + private static final String TARGET = "mv_hits_native"; + + /** Literal q9 (no ORDER BY; rows sorted client-side). The CAST matches PPL's + * double avg semantics — Calcite SQL types AVG(integer) as integer, which + * truncates; the MV path is exact either way (avg_sum is a double column), + * and this validation run CAUGHT the baseline truncating (1551.0 vs + * 1551.333...). */ + private static String q9Direct(String index) { + return "SELECT \"RegionID\", SUM(\"AdvEngineID\"), COUNT(*), AVG(CAST(\"ResolutionWidth\" AS DOUBLE)) FROM " + index + " GROUP BY \"RegionID\""; + } + + /** + * q9 spoken in MV state-column names (the validation-phase hardcoded + * mapping): the target's parquet state docs ARE partial state, so + * SUM-of-sums / SUM-of-counts reproduces every aggregate exactly — + * avg included, because avg_sum ships as a DOUBLE column (float + * division by construction, no cast, no eval, no UDF). + */ + private static String q9Mapped(String index) { + return "SELECT \"RegionID\", SUM(adv_sum), SUM(cnt), SUM(avg_sum) / SUM(avg_cnt) FROM " + index + " GROUP BY \"RegionID\""; + } + + @Override + protected Collection> nodePlugins() { + return List.of( + ArrowBasePlugin.class, + CompositeDataFormatPlugin.class, + MockCommitterEnginePlugin.class, + MVDataFormatPlugin.class, + MVStateDataFormatPlugin.class + ); + } + + @Override + protected Collection additionalNodePlugins() { + return List.of( + classpathPlugin(FlightStreamPlugin.class, List.of(ArrowBasePlugin.class.getName())), + classpathPlugin(AnalyticsPlugin.class, Collections.emptyList()), + classpathPlugin(ParquetOnlyDataFormatPlugin.class, Collections.emptyList()), + classpathPlugin(DataFusionPlugin.class, List.of(AnalyticsPlugin.class.getName())) + ); + } + + private static PluginInfo classpathPlugin(Class pluginClass, List extendedPlugins) { + return new PluginInfo( + pluginClass.getName(), + "classpath plugin", + "NA", + Version.CURRENT, + "1.8", + pluginClass.getName(), + null, + extendedPlugins, + false + ); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG, true) + .put(FeatureFlags.STREAM_TRANSPORT, true) + .build(); + } + + public void testQ9ServedFromStateFilesMatchesDirect() throws Exception { + provision(); + + // Two generations with overlapping groups (the fold has real work). + indexHit(229, 2, 1368); + indexHit(229, 0, 1920); + indexHit(2, 3, 1440); + client().admin().indices().prepareRefresh(SOURCE).get(); + indexHit(229, 7, 1366); // -> avg 1551.333... (non-whole: truncation would show) + indexHit(2, 0, 1600); + client().admin().indices().prepareRefresh(SOURCE).get(); + + // 1. Baseline: q9 DIRECT over the source (raw parquet scan). + List direct = sorted(runner().executeSql(q9Direct(SOURCE))); + + // 2. The moment under test: q9 in state-column terms against the MV + // index — served by the completely standard engine path over the + // target's parquet state docs (partial state in, finals out). + List fromMV = sorted(runner().executeSql(q9Mapped(TARGET))); + + assertEquals("row count", direct.size(), fromMV.size()); + for (int i = 0; i < direct.size(); i++) { + Object[] d = direct.get(i); + Object[] m = fromMV.get(i); + assertEquals("RegionID row " + i, ((Number) d[0]).longValue(), ((Number) m[0]).longValue()); + assertEquals("sum(AdvEngineID) row " + i, ((Number) d[1]).longValue(), ((Number) m[1]).longValue()); + assertEquals("count(*) row " + i, ((Number) d[2]).longValue(), ((Number) m[2]).longValue()); + assertEquals("avg(ResolutionWidth) row " + i, ((Number) d[3]).doubleValue(), ((Number) m[3]).doubleValue(), 1e-9); + } + // The non-whole average is the truncation canary. + Object[] region229 = fromMV.stream().filter(r -> ((Number) r[0]).longValue() == 229L).findFirst().orElseThrow(); + assertEquals(1551.333333, ((Number) region229[3]).doubleValue(), 1e-4); + } + + private void provision() { + client().admin() + .indices() + .prepareCreate(SOURCE) + .setSettings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.pluggable.dataformat.enabled", true) + .put("index.pluggable.dataformat", "composite") + .put("index.composite.primary_data_format", "parquet") + .putList("index.composite.secondary_data_formats", "materialized_view") + .put("index.composite.merge_on_refresh_max_size", "0b") + .put(MVConstants.DEFINITION_SETTING, "clickbench_q9_native") + .putList(MVConstants.SHIP_TARGETS_SETTING, TARGET) + ) + .setMapping("RegionID", "type=integer", "AdvEngineID", "type=integer", "ResolutionWidth", "type=integer") + .get(); + ensureGreen(SOURCE); + + // Target: state fields store the shipped docs (avg_sum is DOUBLE — + // the floating half of avg's state); original-name fields are the + // VALIDATION SURFACE so the literal q9 text plans against this index + // (never populated in docs; the strict read scans only state files). + client().admin() + .indices() + .prepareCreate(TARGET) + .setSettings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.pluggable.dataformat.enabled", true) + .put("index.pluggable.dataformat", "composite") + .put("index.composite.primary_data_format", "parquet") + .putList("index.composite.secondary_data_formats", "mv_state") + .put("index.composite.merge_on_refresh_max_size", "0b") + .put(MVConstants.DEFINITION_SETTING, "clickbench_q9_native") + .put(MVConstants.COLOCATE_WITH_SETTING, SOURCE) + ) + .setMapping( + "RegionID", + "type=integer", + "adv_sum", + "type=long", + "cnt", + "type=long", + "avg_cnt", + "type=long", + "avg_sum", + "type=double", + "AdvEngineID", + "type=integer", + "ResolutionWidth", + "type=integer", + "_mv_source_generation", + "type=long" + ) + .get(); + ensureGreen(TARGET); + } + + private void indexHit(int region, int adv, int width) { + client().prepareIndex(SOURCE).setSource("RegionID", region, "AdvEngineID", adv, "ResolutionWidth", width).get(); + } + + private static List sorted(List rows) { + return rows.stream().sorted(Comparator.comparingLong(r -> ((Number) r[0]).longValue())).toList(); + } + + private SqlPlanRunner runner() { + String node = internalCluster().getNodeNames()[0]; + ClusterService clusterService = internalCluster().getInstance(ClusterService.class, node); + org.opensearch.analytics.exec.DefaultPlanExecutor executor = internalCluster().getInstance( + org.opensearch.analytics.exec.DefaultPlanExecutor.class, + node + ); + return new SqlPlanRunner(clusterService, executor); + } +} diff --git a/sandbox/qa/analytics-engine-rest/build.gradle b/sandbox/qa/analytics-engine-rest/build.gradle index cb822cdf1b0b6..b4220f8335263 100644 --- a/sandbox/qa/analytics-engine-rest/build.gradle +++ b/sandbox/qa/analytics-engine-rest/build.gradle @@ -38,6 +38,7 @@ dependencies { testImplementation project(':sandbox:plugins:dsl-query-executor') testImplementation project(':sandbox:plugins:composite-engine') testImplementation project(':sandbox:plugins:parquet-data-format') + testImplementation project(':sandbox:plugins:mv-data-format') // test-ppl-frontend kept available so ExplainApiIT can hit /_analytics/ppl/_explain; // its profile/stage-timing output isn't matched by the real opensearch-sql plugin's // explain endpoint (returns plain Calcite plan text). All other QA tests use @@ -66,6 +67,9 @@ def configureAnalyticsCluster = { cluster -> cluster.plugin ':sandbox:plugins:analytics-backend-lucene' cluster.plugin ':sandbox:plugins:dsl-query-executor' cluster.plugin ':sandbox:plugins:parquet-data-format' + // MV separate-index track: source materialized_view format + target mv_state fold + cluster.plugin ':sandbox:plugins:mv-data-format' + cluster.plugin ':sandbox:plugins:mv-state-format' // Shim — keeps /_analytics/ppl/_explain available for ExplainApiIT, which asserts on // the shim's profile/stage-timing output that the real plugin doesn't emit. cluster.plugin ':sandbox:plugins:test-ppl-frontend' diff --git a/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MVClickBenchQ9PplIT.java b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MVClickBenchQ9PplIT.java new file mode 100644 index 0000000000000..6fbf7beea6c11 --- /dev/null +++ b/sandbox/qa/analytics-engine-rest/src/test/java/org/opensearch/analytics/qa/MVClickBenchQ9PplIT.java @@ -0,0 +1,203 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.qa; + +import org.opensearch.client.Request; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * ClickBench q9 through the REAL {@code /_plugins/_ppl} API, answered from a + * separate-index materialized view — the full production stack end to end: + * + * 

+ * ingest into SOURCE (composite parquet+lucene+materialized_view)
+ *   -> refresh-time state build (definition's Partial over the flushed parquet)
+ *   -> ship-before-commit to TARGET (composite parquet+lucene+mv_state fold)
+ *   -> PPL fold query over the TARGET's state docs  ==  PPL direct q9 over the SOURCE
+ *
+ * + *

The fold works as plain PPL because MV definitions are pre-decomposed to + * their mergeable core: {@code AVG(ResolutionWidth) = SUM(res_sum)/SUM(cnt)} + * exactly — no approximation, any generation/merge state. + * + *

The direct query is the oracle: both queries run through the same + * opensearch-sql -> Calcite -> analytics-engine -> DataFusion path, only the + * scanned index differs (N state rows on the target vs all docs on the source). + */ +public class MVClickBenchQ9PplIT extends AnalyticsRestTestCase { + + private static final String SOURCE = "hits_mv"; + private static final String TARGET = "mv_hits_q9"; + + /** Batches ingested with a refresh (= generation + ship) after each. */ + private static final int BATCHES = 3; + + public void testQ9FoldOverMVEqualsDirect() throws Exception { + provision(); + + // Direct q9 over the source (oracle). ORDER BY count DESC, then the + // RegionID tiebreak keeps row order deterministic. + Map direct = executePpl( + "source=" + SOURCE + " | stats sum(AdvEngineID) as sum_adv, count() as c, avg(ResolutionWidth) as avg_res by RegionID" + + " | sort - c, RegionID | head 10" + ); + + // q9 from the MV: fold over the target's shipped state docs. + Map fold = executePpl( + "source=" + TARGET + " | stats sum(adv_sum) as sum_adv, sum(cnt) as c, sum(res_sum) as rs by RegionID" + + " | sort - c, RegionID | head 10" + ); + + Map directRows = rowsByRegion(direct, false); + Map foldRows = rowsByRegion(fold, true); + + assertFalse("direct q9 returned no rows", directRows.isEmpty()); + assertEquals("group sets must match", directRows.keySet(), foldRows.keySet()); + for (Map.Entry e : directRows.entrySet()) { + double[] d = e.getValue(); + double[] f = foldRows.get(e.getKey()); + assertEquals("sum_adv for region " + e.getKey(), d[0], f[0], 0.0001); + assertEquals("count for region " + e.getKey(), d[1], f[1], 0.0001); + assertEquals("avg_res for region " + e.getKey(), d[2], f[2], 0.0001); + } + } + + /** + * Extracts {@code region -> [sum_adv, count, avg_res]} from a PPL response. + * For the fold response ({@code sum_adv, c, rs}) the average is computed as + * {@code rs / c} — the exact state algebra the read side performs. + */ + @SuppressWarnings("unchecked") + private static Map rowsByRegion(Map response, boolean foldShape) { + List cols = extractColumnNames(response); + List> datarows = (List>) (List) response.get("datarows"); + int region = cols.indexOf("RegionID"); + int sumAdv = cols.indexOf("sum_adv"); + int cnt = cols.indexOf("c"); + int third = foldShape ? cols.indexOf("rs") : cols.indexOf("avg_res"); + assertTrue("expected columns in " + cols, region >= 0 && sumAdv >= 0 && cnt >= 0 && third >= 0); + Map out = new HashMap<>(); + for (List row : datarows) { + double c = ((Number) row.get(cnt)).doubleValue(); + double thirdVal = ((Number) row.get(third)).doubleValue(); + out.put( + ((Number) row.get(region)).longValue(), + new double[] { ((Number) row.get(sumAdv)).doubleValue(), c, foldShape ? thirdVal / c : thirdVal } + ); + } + return out; + } + + private void provision() throws Exception { + for (String idx : new String[] { SOURCE, TARGET }) { + try { + client().performRequest(new Request("DELETE", "/" + idx)); + } catch (Exception ignored) {} + } + + // 1. SOURCE: composite parquet+lucene+materialized_view, q9 definition, + // shipping state to the target before every commit. + Request createSource = new Request("PUT", "/" + SOURCE); + createSource.setJsonEntity( + "{" + + "\"settings\": {" + + " \"number_of_shards\": 1," + + " \"number_of_replicas\": 0," + + " \"index.pluggable.dataformat.enabled\": true," + + " \"index.pluggable.dataformat\": \"composite\"," + + " \"index.composite.primary_data_format\": \"parquet\"," + + " \"index.composite.secondary_data_formats\": [\"lucene\", \"materialized_view\"]," + + " \"index.mv.definition\": \"clickbench_q9\"," + + " \"index.mv.ship_targets\": [\"" + TARGET + "\"]" + + "}," + // integer (not long): PPL sum(long) lowers to CHECKED_LONG_SUM, + // which has no substrait binding yet (same YAML gap family as + // the boolean min/max case — see MinMaxBooleanAggregationIT). + // Every qa IT sums integer fields; the MV capture is INT64 + // either way (spec columns), so nothing MV-side changes. + + "\"mappings\": { \"dynamic\": \"false\", \"properties\": {" + + " \"RegionID\": { \"type\": \"integer\" }," + + " \"AdvEngineID\": { \"type\": \"integer\" }," + + " \"ResolutionWidth\": { \"type\": \"integer\" }" + + "}}" + + "}" + ); + assertEquals(Boolean.TRUE, assertOkAndParse(client().performRequest(createSource), "create " + SOURCE).get("acknowledged")); + + // 2. TARGET: composite whose mv_state format is the FOLD of the shipped + // state schema; primary colocated with the source primary. + Request createTarget = new Request("PUT", "/" + TARGET); + createTarget.setJsonEntity( + "{" + + "\"settings\": {" + + " \"number_of_shards\": 1," + + " \"number_of_replicas\": 0," + + " \"index.pluggable.dataformat.enabled\": true," + + " \"index.pluggable.dataformat\": \"composite\"," + + " \"index.composite.primary_data_format\": \"parquet\"," + + " \"index.composite.secondary_data_formats\": [\"lucene\", \"mv_state\"]," + + " \"index.mv.definition\": \"clickbench_q9\"," + + " \"index.mv.colocate_with\": \"" + SOURCE + "\"" + + "}," + // dynamic:false + explicit provenance fields: the ship handler + // writes _mv_source_* on every state doc (re-ship idempotency); + // the composite apply path can't do dynamic mapping updates + // (MAPPING_UPDATE_REQUIRED fails the ship, and the flush). + + "\"mappings\": { \"dynamic\": \"false\", \"properties\": {" + + " \"RegionID\": { \"type\": \"integer\" }," + + " \"cnt\": { \"type\": \"integer\" }," + + " \"adv_sum\": { \"type\": \"integer\" }," + + " \"res_sum\": { \"type\": \"integer\" }," + + " \"res_min\": { \"type\": \"integer\" }," + + " \"res_max\": { \"type\": \"integer\" }," + + " \"_mv_source_generation\": { \"type\": \"long\" }" + + "}}" + + "}" + ); + assertEquals(Boolean.TRUE, assertOkAndParse(client().performRequest(createTarget), "create " + TARGET).get("acknowledged")); + + Request health = new Request("GET", "/_cluster/health/" + SOURCE + "," + TARGET); + health.addParameter("wait_for_status", "green"); + health.addParameter("timeout", "60s"); + client().performRequest(health); + + // 3. Ingest deterministic data in BATCHES generations. Regions 1..5; + // region r gets (r * 8) docs per batch with patterned values, so + // every aggregate has a hand-computable expectation and the count + // ordering (5 > 4 > ... > 1) is stable. + for (int b = 0; b < BATCHES; b++) { + StringBuilder bulk = new StringBuilder(); + for (int r = 1; r <= 5; r++) { + for (int i = 0; i < r * 8; i++) { + bulk.append("{\"index\":{}}\n") + .append("{\"RegionID\":") + .append(r) + .append(",\"AdvEngineID\":") + .append((i % 3 == 0) ? r : 0) + .append(",\"ResolutionWidth\":") + .append(1000 + ((b + i) % 7) * 137) + .append("}\n"); + } + } + Request bulkReq = new Request("POST", "/" + SOURCE + "/_bulk"); + bulkReq.setJsonEntity(bulk.toString()); + Map bulkResp = assertOkAndParse(client().performRequest(bulkReq), "bulk batch " + b); + assertEquals("bulk batch " + b + " must not error", Boolean.FALSE, bulkResp.get("errors")); + + // Refresh = generation flush = state build + ship-before-commit. + client().performRequest(new Request("POST", "/" + SOURCE + "/_refresh")); + } + // No target refresh: the ship ack already certified searchability + // (refresh-before-ack on the target primary) — asserting without one + // is part of the contract under test. + } +} diff --git a/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java b/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java index 7ec6a578cfe5d..72d9cf6ce7aa7 100644 --- a/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java @@ -103,6 +103,7 @@ import java.io.Closeable; import java.io.IOException; +import java.io.UncheckedIOException; import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.Collections; @@ -1193,7 +1194,12 @@ public void flush(boolean force, boolean waitIfOngoing) throws EngineException { // and available to the deletion policy when onCommit is triggered. translogManager.ensureCanFlush(); translogManager.syncTranslog(); - Map commitData = new HashMap<>(); + // Cross-index commit coordination (decision 25): + // format engines may trigger dependent commits and + // contribute commit meta; failure refuses THIS commit. + Map participantEntries = indexingExecutionEngine.beforeCommit(); + + Map commitData = new HashMap<>(participantEntries); commitData.put( CatalogSnapshot.LAST_COMPOSITE_WRITER_GEN_KEY, Long.toString(snapshot.getLastWriterGeneration()) @@ -1473,6 +1479,20 @@ public void onMergesDrained(Runnable listener) { mergeScheduler.onDrained(listener); } + /** + * Version of the latest (possibly uncommitted) catalog snapshot. Used by + * cross-index commit coordination (a ship ack reports the version its + * apply produced; the source commits only after this version is durable + * on the target — decision 25). + */ + public long latestCatalogSnapshotVersion() { + try (GatedConditionalCloseable ref = catalogSnapshotManager.acquireSnapshotForCommit()) { + return ref.get().getVersion(); + } catch (IOException e) { + throw new UncheckedIOException("failed to read latest catalog snapshot version", e); + } + } + /** * Returns the number of currently active (in-flight) merge tasks. * diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormat.java b/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormat.java index ed261b976403f..73957da9009d2 100644 --- a/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormat.java +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormat.java @@ -46,6 +46,40 @@ public abstract class DataFormat { */ public abstract Set supportedFields(); + /** + * Whether this format is exempt from the per-segment cross-format row-count + * parity checks. Formats that store the same logical rows as other formats + * (the default) must report equal row counts per segment. A derived + * format — one whose files are computed from another format's data rather + * than ingested row-by-row (e.g. a materialized-view aggregate, one row per + * group) — reports its own row count and is exempt. + * + *

Exempt formats must never be used as the source of segment-ordinal to + * leaf mappings or merge-policy row accounting; those always come from a + * non-exempt (primary) format. + * + * @return true if this format's per-segment row count may differ from other formats' + */ + public boolean exemptFromRowParity() { + return false; + } + + /** + * Whether this format may legally produce NO files for a generation whose + * primary flushed. False for regular formats — the composite flush contract + * is files-for-all-formats-or-none, so a missing file set means data loss. + * Derived formats whose output lives elsewhere return true: a format that + * ships its state to an external location (e.g. a separate MV index, + * ship-before-commit) has nothing to register in this index's catalog, and + * a derived format may also skip a generation it will cover later + * (skip-and-backfill). + * + * @return true if an empty flush result is legal for this format + */ + public boolean mayEmitNoFiles() { + return false; + } + @Override public final boolean equals(Object o) { if (this == o) return true; diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/DerivedDataFormat.java b/server/src/main/java/org/opensearch/index/engine/dataformat/DerivedDataFormat.java new file mode 100644 index 0000000000000..95a1ebabc0650 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/DerivedDataFormat.java @@ -0,0 +1,55 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.engine.dataformat; + +import java.util.Set; + +/** + * Base type for DERIVED data formats: formats whose per-segment output is + * computed FROM the ingested documents rather than being a storage of them + * (e.g. materialized-view aggregate state). The type codifies the contract + * that individual overrides previously expressed piecemeal: + * + *

    + *
  • Row parity exempt — derived output is per-group, not per-doc; + * its row counts legally differ from storage formats'.
    • + *
  • May emit no files — a derived format may skip a generation + * (skip-and-backfill) or ship its output elsewhere entirely (the + * separate-index MV target); an empty flush result is legal and the + * composite engine treats the format as optional per segment.
    • + *
  • Never claims fields — derived formats observe the composite + * broadcast; they do not participate in capability assignment and never + * win a claim ({@code priority()} = MAX, {@code supportedFields()} + * empty).
    • + *
+ * + * @opensearch.experimental + */ +public abstract class DerivedDataFormat extends DataFormat { + + @Override + public final long priority() { + return Long.MAX_VALUE; + } + + @Override + public final Set supportedFields() { + return Set.of(); + } + + @Override + public final boolean exemptFromRowParity() { + return true; + } + + @Override + public final boolean mayEmitNoFiles() { + return true; + } +} diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/IndexingExecutionEngine.java b/server/src/main/java/org/opensearch/index/engine/dataformat/IndexingExecutionEngine.java index bf511c0c7e32f..ad70fc0d9b3bc 100644 --- a/server/src/main/java/org/opensearch/index/engine/dataformat/IndexingExecutionEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/IndexingExecutionEngine.java @@ -120,6 +120,22 @@ public interface IndexingExecutionEngine beforeCommit() throws IOException { + return java.util.Map.of(); + } + default FormatChecksumStrategy getChecksumStrategy() { return null; } diff --git a/server/src/main/java/org/opensearch/index/engine/exec/coord/CatalogSnapshotManager.java b/server/src/main/java/org/opensearch/index/engine/exec/coord/CatalogSnapshotManager.java index b1416e240b39f..e276443a5eaa6 100644 --- a/server/src/main/java/org/opensearch/index/engine/exec/coord/CatalogSnapshotManager.java +++ b/server/src/main/java/org/opensearch/index/engine/exec/coord/CatalogSnapshotManager.java @@ -31,6 +31,7 @@ import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Locale; @@ -590,15 +591,26 @@ private Segment getSegment(Map writerFileSetMap) { if (writerFileSetMap.isEmpty()) { throw new IllegalArgumentException("writerFileSetMap must not be empty"); } - // Check for null values (format participated but returned no result) + // A null value means the format participated but produced no result. + // For mayEmitNoFiles formats that is a LEGAL outcome (e.g. a derived + // format in ship mode treats merges as a non-event — nothing local to + // produce); for all other formats it means the merge was incomplete. + Map present = new HashMap<>(); for (Map.Entry entry : writerFileSetMap.entrySet()) { if (entry.getValue() == null) { + if (entry.getKey().mayEmitNoFiles()) { + continue; + } throw new IllegalStateException("WriterFileSet is null for format [" + entry.getKey().name() + "] — merge was incomplete"); } + present.put(entry.getKey(), entry.getValue()); + } + if (present.isEmpty()) { + throw new IllegalStateException("merge produced no files for any format — nothing to register"); } - long generation = writerFileSetMap.values().iterator().next().writerGeneration(); + long generation = present.values().iterator().next().writerGeneration(); Segment.Builder segment = Segment.builder(generation); - for (Map.Entry entry : writerFileSetMap.entrySet()) { + for (Map.Entry entry : present.entrySet()) { segment.addSearchableFiles(entry.getKey(), entry.getValue()); } return segment.build(); @@ -665,15 +677,26 @@ private boolean assertSegmentGenerationFileConsistency(List newSegments * where rows are silently dropped or duplicated during merge. */ private boolean assertRowCountConservation(Set sourceSegments, Segment mergedSegment) { + // Derived formats (row-parity exempt) are excluded on BOTH sides: + // their merge is a recompute/fold, so the merged state legitimately + // has FEWER rows than the inputs (e.g. mv_state folding shipped + // state across generations). Conservation is asserted only for + // formats that carry the documents themselves. long sourceRows = 0; for (Segment seg : sourceSegments) { - for (WriterFileSet wfs : seg.dfGroupedSearchableFiles().values()) { - sourceRows += wfs.numRows(); + for (Map.Entry entry : seg.dfGroupedSearchableFiles().entrySet()) { + if (ROW_PARITY_EXEMPT_FORMATS.contains(entry.getKey())) { + continue; + } + sourceRows += entry.getValue().numRows(); } } long mergedRows = 0; - for (WriterFileSet wfs : mergedSegment.dfGroupedSearchableFiles().values()) { - mergedRows += wfs.numRows(); + for (Map.Entry entry : mergedSegment.dfGroupedSearchableFiles().entrySet()) { + if (ROW_PARITY_EXEMPT_FORMATS.contains(entry.getKey())) { + continue; + } + mergedRows += entry.getValue().numRows(); } if (sourceRows != mergedRows) { logger.error("Row count mismatch: source segments have {} rows but merged segment has {} rows", sourceRows, mergedRows); @@ -694,11 +717,19 @@ private boolean assertRowCountConservation(Set sourceSegments, Segment * single refresh — which produces silent correctness issues like different counts * from {@code match} vs {@code LIKE} over the same field. */ + // POC(mv): formats exempt from row parity — derived formats whose rows are aggregates + // of the primary's rows. TODO(mv): replace with DataFormat.exemptFromRowParity() once + // the registry is reachable from here (needs plumbing through the engine constructor). + private static final java.util.Set ROW_PARITY_EXEMPT_FORMATS = java.util.Set.of("materialized_view", "mv_state"); + private void verifyPerSegmentCrossFormatRowCountParity(List segments) { for (Segment seg : segments) { long expected = -1L; String referenceFormat = null; for (Map.Entry entry : seg.dfGroupedSearchableFiles().entrySet()) { + if (ROW_PARITY_EXEMPT_FORMATS.contains(entry.getKey())) { + continue; + } long rows = entry.getValue().numRows(); if (expected == -1L) { expected = rows; diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index c0ee282d2eb7f..d592ef7eb4624 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -4437,6 +4437,21 @@ private void doCheckIndex() throws IOException { recoveryState.getVerifyIndex().checkIndexTime(Math.max(0, TimeValue.nsecToMSec(System.nanoTime() - timeNS))); } + /** + * Latest catalog snapshot version of this shard's composite engine, or + * {@code -1} if the shard is not composite-backed. Used by cross-index + * commit coordination (the MV commit sync, decision 25): a ship ack + * reports the version its apply produced; the source commits only after + * that version is durable here. + */ + public long compositeCatalogSnapshotVersion() { + Indexer indexer = getIndexerOrNull(); + if (indexer instanceof org.opensearch.index.engine.DataFormatAwareEngine dfe) { + return dfe.latestCatalogSnapshotVersion(); + } + return -1L; + } + Indexer getIndexer() { Indexer engine = getIndexerOrNull(); if (engine == null) {