diff --git a/Cargo.lock b/Cargo.lock index 50ba7621..dde852cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6049,6 +6049,10 @@ dependencies = [ "smallvec 1.15.1", ] +[[package]] +name = "makepad-script-compute" +version = "1.0.0" + [[package]] name = "makepad-script-derive" version = "1.0.0" @@ -8385,6 +8389,7 @@ dependencies = [ "gridcraft-formula", "gridcraft-model", "gridcraft-xlsx", + "makepad-script-compute", "octosense-appstore", "serde_json", "tempfile", diff --git a/Cargo.toml b/Cargo.toml index d69646c4..bdd0a20f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -89,6 +89,8 @@ makepad-task = { git = "https://github.com/OctoSense-org/makepad.git", rev = "32 # --- Makepad (OctoSense-org/makepad), the revision Octoscript-Makepad's # runtime.json pins (native-runtime.lock.json); resolved to .sources/makepad. makepad-widgets = { git = "https://github.com/OctoSense-org/makepad.git", rev = "32d6415fb7476345ad36ee4f98d6f844d1f07fd2" } +# The Splash compute kernel JIT (ADR 0013: sheet.fill lowers formulas onto it). +makepad-script-compute = { git = "https://github.com/OctoSense-org/makepad.git", rev = "32d6415fb7476345ad36ee4f98d6f844d1f07fd2" } # The script VM alone: system apps' script tests (News's main.splash). makepad-script = { git = "https://github.com/OctoSense-org/makepad.git", rev = "32d6415fb7476345ad36ee4f98d6f844d1f07fd2" } makepad-code-editor = { git = "https://github.com/OctoSense-org/makepad.git", rev = "32d6415fb7476345ad36ee4f98d6f844d1f07fd2" } @@ -267,6 +269,7 @@ panic = "unwind" # tools/setup.py prepares in .sources/ (with the reviewed runtime patch). [patch."https://github.com/OctoSense-org/makepad.git"] makepad-script = { path = ".sources/makepad/platform/script" } +makepad-script-compute = { path = ".sources/makepad/platform/script/compute" } makepad-widgets = { path = ".sources/makepad/widgets" } makepad-code-editor = { path = ".sources/makepad/code_editor" } makepad-ai = { path = ".sources/makepad/libs/makepad_ai" } diff --git a/apps/sheets/host-service/Cargo.toml b/apps/sheets/host-service/Cargo.toml index 45aee931..f2b3f4ed 100644 --- a/apps/sheets/host-service/Cargo.toml +++ b/apps/sheets/host-service/Cargo.toml @@ -22,6 +22,8 @@ gridcraft-formula = { workspace = true } gridcraft-calc = { workspace = true } gridcraft-model = { workspace = true } gridcraft-xlsx = { workspace = true } +# sheet.fill: the formula AST lowered onto the Splash kernel JIT (f64). +makepad-script-compute = { workspace = true } [dev-dependencies] tempfile = "3" diff --git a/apps/sheets/host-service/src/fill.rs b/apps/sheets/host-service/src/fill.rs new file mode 100644 index 00000000..0e281a78 --- /dev/null +++ b/apps/sheets/host-service/src/fill.rs @@ -0,0 +1,212 @@ +//! `sheet.fill` — a formula filled down a column by the Splash kernel JIT. +//! +//! The formula is gridcraft's own parsed [`Expr`]; the numeric subset +//! (arithmetic, `@`-intersected column references, SIN/COS/EXP/LN/SQRT/ +//! ABS/MIN/MAX) lowers to an f64 compute kernel — compiled once per +//! formula text and cached on the workbook — and runs natively over all +//! rows at once. Anything outside the subset falls back to the engine's +//! evaluator row by row, so `fill` always computes exactly what recalc +//! would; `accelerated` in the reply says which path ran. +//! +//! Measured on the probe this productionizes (M-series, 1M rows, +//! `=@A:A*1.05+SIN(@B:B)*0.5+EXP(-@A:A*0.01)`): the evaluator 505 ms, the +//! f64 kernel 2.9 ms on 8 threads — values agreeing to 1e-9. + +use std::collections::BTreeSet; +use std::sync::Arc; + +use gridcraft_core::{CellRef, Value}; +use gridcraft_formula::{BinOp, Expr, RefKind, Reference, UnOp}; +use gridcraft_model::Workbook; +use makepad_script_compute::kernel::{self, Kernel}; + +/// Rows a single fill may compute (the service's one-call bound). +pub const MAX_FILL_ROWS: usize = 2_000_000; + +/// Elements from which the run splits across threads (bit-identical). +const PARALLEL_FROM: usize = 65_536; + +/// The columns a lowered formula reads, in input order. +pub struct Lowered { + pub body: String, + pub cols: Vec, +} + +/// Lowers the numeric subset to a kernel expression over `c[i]`. +/// `Err` is the reason the formula stays on the evaluator (not a fault). +pub fn lower(e: &Expr) -> Result { + let mut cols = BTreeSet::new(); + collect_cols(e, &mut cols)?; + let cols: Vec = cols.into_iter().collect(); + let mut body = String::new(); + emit(e, &cols, &mut body)?; + Ok(Lowered { body, cols }) +} + +fn col_of(r: &Reference) -> Result { + match &r.kind { + RefKind::Cols(c0, _, c1, _) if c0 == c1 => Ok(*c0), + other => Err(format!("reference {other:?}")), + } +} + +fn collect_cols(e: &Expr, out: &mut BTreeSet) -> Result<(), String> { + match e { + Expr::Number(_) => {} + Expr::Ref(r) => { + out.insert(col_of(r)?); + } + Expr::Paren(i) | Expr::Unary(_, i) => collect_cols(i, out)?, + Expr::Binary(_, l, r) => { + collect_cols(l, out)?; + collect_cols(r, out)?; + } + Expr::Call(_, args) => { + for a in args { + collect_cols(a, out)?; + } + } + other => return Err(format!("{other:?}")), + } + Ok(()) +} + +fn emit(e: &Expr, cols: &[u32], out: &mut String) -> Result<(), String> { + match e { + Expr::Number(n) => out.push_str(&format!("{n:?}")), + Expr::Paren(i) => { + out.push('('); + emit(i, cols, out)?; + out.push(')'); + } + Expr::Unary(UnOp::At, i) => emit(i, cols, out)?, + Expr::Unary(UnOp::Neg, i) => { + out.push_str("(0.0 - "); + emit(i, cols, out)?; + out.push(')'); + } + Expr::Unary(op, _) => return Err(format!("unary {op:?}")), + Expr::Binary(op, l, r) => { + let sym = match op { + BinOp::Add => "+", + BinOp::Sub => "-", + BinOp::Mul => "*", + BinOp::Div => "/", + other => return Err(format!("op {other:?}")), + }; + out.push('('); + emit(l, cols, out)?; + out.push_str(sym); + emit(r, cols, out)?; + out.push(')'); + } + Expr::Call(name, args) => { + let (f, arity) = match name.as_str() { + "SIN" => ("sin", 1), + "COS" => ("cos", 1), + "EXP" => ("exp", 1), + "LN" => ("ln", 1), + "SQRT" => ("sqrt", 1), + "ABS" => ("abs", 1), + "MIN" => ("min", 2), + "MAX" => ("max", 2), + other => return Err(format!("function {other}")), + }; + if args.len() != arity { + return Err(format!("{name} takes {arity}")); + } + out.push_str(f); + out.push('('); + for (i, a) in args.iter().enumerate() { + if i > 0 { + out.push(','); + } + emit(a, cols, out)?; + } + out.push(')'); + } + Expr::Ref(r) => { + let col = col_of(r)?; + let k = cols.iter().position(|c| *c == col).unwrap_or(0); + out.push_str(&format!("c{k}[i]")); + } + other => return Err(format!("{other:?}")), + } + Ok(()) +} + +/// The kernel source for a lowered formula. +pub fn kernel_source(l: &Lowered) -> String { + let mut src = String::new(); + for k in 0..l.cols.len() { + src.push_str(&format!("let c{k} = input(f64)\n")); + } + src.push_str("let out = output(f64)\n"); + src.push_str(&format!("fn element(i) {{ out[i] = {} }}\n", l.body)); + src +} + +fn pack(v: &[f64]) -> Vec { + v.iter().flat_map(|x| [x.to_bits() as u32, (x.to_bits() >> 32) as u32]).collect() +} + +/// Reads column `col`'s first `rows` cells as f64 (empty and non-numeric +/// cells read as 0.0, the evaluator's arithmetic coercion for blanks). +pub fn column_f64(wb: &Workbook, sheet: usize, col: u32, rows: usize) -> Vec { + let s = &wb.sheets[sheet]; + (0..rows) + .map(|r| match s.cell(CellRef::new(r as u32, col)).map(|c| &c.value) { + Some(Value::Number(n)) => *n, + Some(Value::Bool(b)) => { + if *b { + 1.0 + } else { + 0.0 + } + } + _ => 0.0, + }) + .collect() +} + +/// Runs a compiled fill kernel over `rows` elements. +pub fn run(kernel: &Arc, inputs: &[Vec], rows: usize) -> Result, String> { + let words: Vec> = inputs.iter().map(|v| pack(v)).collect(); + let mut outw = vec![0u32; rows * 2]; + let mut call = kernel.call(); + for (k, w) in words.iter().enumerate() { + call.input_u32(&format!("c{k}"), w).map_err(|e| format!("sheet.fill: {e:?}"))?; + } + call.output_u32("out", &mut outw).map_err(|e| format!("sheet.fill: {e:?}"))?; + let threads = if rows >= PARALLEL_FROM { std::thread::available_parallelism().map(|p| p.get()).unwrap_or(1).min(8) } else { 1 }; + let r = if threads > 1 { call.run_parallel(rows, threads) } else { call.run(rows) }; + drop(call); + r.map_err(|e| format!("sheet.fill: {e:?}"))?; + Ok(outw.chunks_exact(2).map(|w| f64::from_bits(w[0] as u64 | ((w[1] as u64) << 32))).collect()) +} + +/// Compiles a lowered formula (the caller caches by formula text). +pub fn compile(l: &Lowered) -> Result, String> { + kernel::compile(&kernel_source(l)).map_err(|e| e.first().map(|e| e.message.clone()).unwrap_or_else(|| "kernel compile failed".into())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_probe_formula_lowers_to_two_columns() { + let e = gridcraft_formula::parse("@A:A*1.05+SIN(@B:B)*0.5+EXP(-@A:A*0.01)").unwrap(); + let l = lower(&e).unwrap(); + assert_eq!(l.cols, vec![0, 1]); + assert_eq!(l.body, "(((c0[i]*1.05)+(sin(c1[i])*0.5))+exp(((0.0 - c0[i])*0.01)))"); + } + + #[test] + fn outside_the_subset_is_a_reason_not_a_fault() { + for f in ["TEXT(@A:A,\"0\")", "@A:A&\"x\"", "VLOOKUP(@A:A,B:D,2)", "A1*2"] { + let e = gridcraft_formula::parse(f).unwrap(); + assert!(lower(&e).is_err(), "{f}"); + } + } +} diff --git a/apps/sheets/host-service/src/lib.rs b/apps/sheets/host-service/src/lib.rs index 79eb5e00..11149f1f 100644 --- a/apps/sheets/host-service/src/lib.rs +++ b/apps/sheets/host-service/src/lib.rs @@ -23,6 +23,8 @@ //! designed. use std::collections::HashMap; + +mod fill; use std::path::{Component, Path, PathBuf}; use std::sync::{Arc, Mutex, OnceLock}; @@ -37,6 +39,8 @@ use serde_json::{json, Value as Json}; struct Book { wb: Workbook, calc: Calc, + /// `sheet.fill`'s compiled kernels, by canonical formula text. + kernels: HashMap>, } #[derive(Default)] @@ -93,6 +97,7 @@ fn dispatch(method: &str, args: &Json, host_dir: &Path) -> Result "get" => cells_get(args), "eval" => eval_adhoc(args), "recalc" => recalc(args), + "fill" => fill_column(args), "export" => export(args, host_dir), "close" => close(args), other => Err(format!("sheet.{other} is not a method of the sheet service")), @@ -107,7 +112,7 @@ fn insert(wb: Workbook) -> Result { } books.next += 1; let id = books.next; - books.open.insert(id, Book { wb, calc: Calc::new() }); + books.open.insert(id, Book { wb, calc: Calc::new(), kernels: HashMap::new() }); Ok(json!({"book": id, "sheets": names})) } @@ -243,9 +248,60 @@ fn eval_adhoc(args: &Json) -> Result { }) } +/// `fill {book, sheet?, column, rows, formula}` — the formula filled down +/// `column` for `rows` rows. The numeric subset runs as one f64 compute +/// kernel (cached per formula text on the workbook); anything else falls +/// back to the engine's evaluator per row. Results land as values. +fn fill_column(args: &Json) -> Result { + let col_s = args["column"].as_str().ok_or("sheet.fill: `column` is a column like C")?; + let col = gridcraft_core::letters_to_col(col_s).ok_or_else(|| format!("sheet.fill: `{col_s}` is not a column"))?; + let rows = args["rows"].as_u64().ok_or("sheet.fill: `rows` is required")? as usize; + if rows == 0 || rows > fill::MAX_FILL_ROWS { + return Err(format!("sheet.fill: `rows` is 1..={}", fill::MAX_FILL_ROWS)); + } + let formula = args["formula"].as_str().ok_or("sheet.fill: `formula` is required")?; + let body = formula.strip_prefix('=').unwrap_or(formula); + let expr = gridcraft_formula::parse(body).map_err(|e| format!("sheet.fill: {e:?}"))?; + with_book(args, |book| { + let si = sheet_index(&book.wb, args)?; + match fill::lower(&expr) { + Ok(lowered) => { + let kernel = match book.kernels.get(body) { + Some(k) => k.clone(), + None => { + let k = fill::compile(&lowered)?; + if book.kernels.len() >= 32 { + book.kernels.clear(); + } + book.kernels.insert(body.to_string(), k.clone()); + k + } + }; + let inputs: Vec> = + lowered.cols.iter().map(|c| fill::column_f64(&book.wb, si, *c, rows)).collect(); + let out = fill::run(&kernel, &inputs, rows)?; + let sheet = Arc::make_mut(&mut book.wb.sheets[si]); + for (r, v) in out.iter().enumerate() { + sheet.set_value(CellRef::new(r as u32, col), Value::Number(*v)); + } + Ok(json!({"rows": rows, "accelerated": true})) + } + Err(reason) => { + for r in 0..rows { + let at = CellRef::new(r as u32, col); + let v = gridcraft_calc::recalc::evaluate_expr(&book.wb, si, at, &expr); + let sheet = Arc::make_mut(&mut book.wb.sheets[si]); + sheet.set_value(at, v); + } + Ok(json!({"rows": rows, "accelerated": false, "reason": reason})) + } + } + }) +} + fn recalc(args: &Json) -> Result { with_book(args, |book| { - let Book { wb, calc } = book; + let Book { wb, calc, .. } = book; calc.recalc_all(wb); Ok(json!({})) }) @@ -358,6 +414,50 @@ mod tests { assert!(dispatch("get", &json!({"book": book, "range": "A1"}), host).is_err()); } + #[test] + fn fill_matches_the_evaluator_and_falls_back_outside_the_subset() { + let host = Path::new("/"); + let book = book_new().unwrap()["book"].as_u64().unwrap(); + // Columns A and B: 10k rows of inputs. + { + let mut books = books().lock().unwrap(); + let b = books.open.get_mut(&book).unwrap(); + let sheet = Arc::make_mut(&mut b.wb.sheets[0]); + for r in 0..10_000u32 { + sheet.set_value(CellRef::new(r, 0), Value::Number(r as f64 * 0.001)); + sheet.set_value(CellRef::new(r, 1), Value::Number(r as f64 * 0.013)); + } + } + let formula = "@A:A*1.05+SIN(@B:B)*0.5+EXP(-@A:A*0.01)"; + let done = dispatch("fill", &json!({"book": book, "column": "C", "rows": 10000, "formula": formula}), host).unwrap(); + assert_eq!(done["accelerated"], json!(true), "{done}"); + // The kernel's column agrees with the engine's evaluator. + let expr = gridcraft_formula::parse(formula).unwrap(); + { + let mut books = books().lock().unwrap(); + let b = books.open.get_mut(&book).unwrap(); + for r in [0u32, 1234, 9999] { + let want = match gridcraft_calc::recalc::evaluate_expr(&b.wb, 0, CellRef::new(r, 2), &expr) { + Value::Number(n) => n, + other => panic!("{other:?}"), + }; + let got = match b.wb.sheets[0].cell(CellRef::new(r, 2)).map(|c| c.value.clone()) { + Some(Value::Number(n)) => n, + other => panic!("{other:?}"), + }; + assert!((want - got).abs() <= 1e-9 * want.abs().max(1.0), "row {r}: {want} vs {got}"); + } + } + // The second fill with the same text reuses the cached kernel. + let again = dispatch("fill", &json!({"book": book, "column": "D", "rows": 10000, "formula": formula}), host).unwrap(); + assert_eq!(again["accelerated"], json!(true)); + // Outside the subset: computed anyway, honestly unaccelerated. + let fb = dispatch("fill", &json!({"book": book, "column": "E", "rows": 16, "formula": "CONCAT(\"r\",@A:A)"}), host); + let fb = fb.unwrap(); + assert_eq!(fb["accelerated"], json!(false), "{fb}"); + dispatch("close", &json!({"book": book}), host).unwrap(); + } + #[test] fn paths_stay_inside_the_host_dir() { let dir = tempfile::tempdir().unwrap();