diff --git a/README.md b/README.md index 07c94b3fb..c9d56b787 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,10 @@ One binary, seven seats; the file's architecture picks the seat, and `--place` c The seventh seat, `nemotron`, serves Nemotron 3 Nano 4B (new in 0.3.3) and Nemotron 3.5 Bolt 74B-A9B (new in 0.3.4) from a file on disk, tested as `NVIDIA-Nemotron3-Nano-4B-Q4_K_M.gguf` and `NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf`: `bloomery-serve -m PATH/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB.gguf`. -The table above lists the seats with a checked `--hf` row. +The table above lists the seats with a checked `--hf` row. The qwen38 seat also serves NVIDIA's NVFP4 checkpoint +folder as downloaded, with no conversion, on a card that holds every routed expert such as a DGX Spark's GB10: +`bloomery-serve -m PATH/Qwen3.8-Flash-Next-NVFP4` or `bloomery-serve --hf nvidia/Qwen3.8-Flash-Next-NVFP4` +([`docs/serving.md`](docs/serving.md#the-seats)). ```sh curl -s http://127.0.0.1:8080/v1/chat/completions \ diff --git a/crates/engram/src/hash.rs b/crates/engram/src/hash.rs index 45b1d9164..6a1d0681f 100644 --- a/crates/engram/src/hash.rs +++ b/crates/engram/src/hash.rs @@ -6,7 +6,7 @@ //! nothing in this module has a default, because a default would be a number //! nobody measured. Two key sets exist, each read by its own constructor: //! [`Hash::from_gguf`] (V4.1's engram, `deepseek41.engram.*`) and -//! [`Hash::ple_from_gguf`] (a PLE site, `.ple.*`). +//! [`Hash::ple_from_keys`] (a PLE site, `.ple.*`). //! //! The formula, per site `e` and per token ([`ngram::rows`]): //! @@ -69,7 +69,7 @@ pub enum Window { /// The hash constants of one model, one set per site. /// -/// Built by [`Hash::from_gguf`] or [`Hash::ple_from_gguf`] from a shard +/// Built by [`Hash::from_gguf`] or [`Hash::ple_from_keys`] from a shard /// header; nothing here is derived and nothing has a default. pub struct Hash { /// Block index of each site, in the order the metadata lists them. This is @@ -97,7 +97,7 @@ impl Hash { /// key rather than a set of constants that is silently short. pub fn from_gguf(meta: &Inventory) -> Result { let k = |suffix: &str| format!("{ENGRAM_PREFIX}{suffix}"); - let layer_ids: Vec = ints(meta, &k("layer_ids"))? + let layer_ids: Vec = ints(&meta.meta, &k("layer_ids"))? .into_iter() .map(|v| u32::try_from(v).map_err(|_| value_err(k("layer_ids"), "a block index"))) .collect::>()?; @@ -106,9 +106,9 @@ impl Hash { return Err(value_err(k("layer_ids"), "at least one engram site")); } - let n_heads = usize::try_from(scalar(meta, &k("head_count"))?) + let n_heads = usize::try_from(scalar(&meta.meta, &k("head_count"))?) .map_err(|_| value_err(k("head_count"), "a head count"))?; - let n_gram = usize::try_from(scalar(meta, &k("max_ngram_size"))?) + let n_gram = usize::try_from(scalar(&meta.meta, &k("max_ngram_size"))?) .map_err(|_| value_err(k("max_ngram_size"), "an n-gram size"))?; // The port's own floor: one head and a 2-gram, because `n_cols` is // `(n_gram - 1) * n_heads` and a site with no buckets reads no rows. @@ -120,15 +120,15 @@ impl Hash { } let n_cols = (n_gram - 1) * n_heads; - let key_length = u32::try_from(scalar(meta, &k("key_length"))?) + let key_length = u32::try_from(scalar(&meta.meta, &k("key_length"))?) .map_err(|_| value_err(k("key_length"), "a key length"))?; - let pad = scalar(meta, &k("pad_id"))?; + let pad = scalar(&meta.meta, &k("pad_id"))?; - let mult = ints(meta, &k("multipliers"))?; + let mult = ints(&meta.meta, &k("multipliers"))?; need_len(k("multipliers"), mult.len(), sites * n_gram)?; - let prime = ints(meta, &k("primes"))?; + let prime = ints(&meta.meta, &k("primes"))?; need_len(k("primes"), prime.len(), sites * n_cols)?; - let offset = ints(meta, &k("offsets"))?; + let offset = ints(&meta.meta, &k("offsets"))?; need_len(k("offsets"), offset.len(), sites * n_cols)?; check_buckets( &prime, @@ -138,7 +138,7 @@ impl Hash { "a bucket top inside u32", )?; - let token_map: Vec = ints(meta, &k("token_map"))? + let token_map: Vec = ints(&meta.meta, &k("token_map"))? .into_iter() .map(|v| { u32::try_from(v).map_err(|_| value_err(k("token_map"), "a compressed vocab id")) @@ -161,8 +161,9 @@ impl Hash { }) } - /// Read a PLE site's constants out of the header of the shard that - /// carries `.ple.*` (the first shard of a split set). + /// Read a PLE site's constants out of the metadata pairs of the header + /// that carries `.ple.*` (`gguf::Split::keys`: the first shard of a + /// split set, or a checkpoint folder's header). /// /// The keys, as the port's loader reads them (qwen4exp.cpp:66-121): /// `ple.layers` (exactly one site), `ple.ngram_size` (2 to 8), @@ -178,7 +179,11 @@ impl Hash { /// states. The EOS is the file's `ple.eos_token_id`, never the /// tokenizer's end of text: the two differ (qwen4exp: 248,044 against /// 248,046). - pub fn ple_from_gguf(meta: &Inventory, arch: &str, n_vocab: u32) -> Result { + pub fn ple_from_keys( + meta: &[(String, Value)], + arch: &str, + n_vocab: u32, + ) -> Result { let k = |suffix: &str| format!("{arch}.ple.{suffix}"); let layer_ids: Vec = ints(meta, &k("layers"))? .into_iter() @@ -223,7 +228,7 @@ impl Hash { .ok_or_else(|| value_err(k(suffix), "a token of the vocabulary")) }; let eos = token("eos_token_id")?; - let image = match meta.value(&k("image_token_id")) { + let image = match value(meta, &k("image_token_id")) { Some(_) => Some(token("image_token_id")?), None => None, }; @@ -391,7 +396,7 @@ impl Hash { /// # Panics /// /// On a raw-id hash ([`Window::Eos`]), which has no map; its vocabulary - /// is the one [`Hash::ple_from_gguf`] was given. + /// is the one [`Hash::ple_from_keys`] was given. pub fn check_vocab(&self, n_vocab: usize) -> Result<(), EngramError> { let map = self.mapped("check_vocab").0; if map.len() == n_vocab { @@ -646,12 +651,15 @@ fn unsigned(v: &Value) -> Option { } } +/// `key`'s value among `meta`'s pairs, the first if it repeats (`Inventory::value`'s rule). +fn value<'m>(meta: &'m [(String, Value)], key: &str) -> Option<&'m Value> { + meta.iter().find(|(k, _)| k == key).map(|(_, v)| v) +} + /// One mandatory unsigned scalar. -fn scalar(meta: &Inventory, key: &str) -> Result { +fn scalar(meta: &[(String, Value)], key: &str) -> Result { let key = key.to_string(); - let v = meta - .value(&key) - .ok_or_else(|| EngramError::MissingKey { key: key.clone() })?; + let v = value(meta, &key).ok_or_else(|| EngramError::MissingKey { key: key.clone() })?; unsigned(v).ok_or(EngramError::KeyType { key, want: "a non-negative integer", @@ -659,11 +667,9 @@ fn scalar(meta: &Inventory, key: &str) -> Result { } /// One mandatory array of unsigned integers, widened to `u64`. -fn ints(meta: &Inventory, key: &str) -> Result, EngramError> { +fn ints(meta: &[(String, Value)], key: &str) -> Result, EngramError> { let key = key.to_string(); - let v = meta - .value(&key) - .ok_or_else(|| EngramError::MissingKey { key: key.clone() })?; + let v = value(meta, &key).ok_or_else(|| EngramError::MissingKey { key: key.clone() })?; let Value::Array(items) = v else { return Err(EngramError::KeyType { key, @@ -1080,7 +1086,7 @@ mod tests { /// tokens at position 9. #[test] fn ple_rows_are_iks() { - let hash = Hash::ple_from_gguf(&ple_fixture(), "qwen4exp", 248_320).unwrap(); + let hash = Hash::ple_from_keys(&ple_fixture().meta, "qwen4exp", 248_320).unwrap(); assert_eq!((hash.sites(), hash.n_gram(), hash.n_cols()), (1, 3, 16)); let want: Vec = PLE_IK_ROWS.iter().flatten().copied().collect(); @@ -1108,7 +1114,7 @@ mod tests { /// another length, and a V4.1 hash asked for raw-id rows. #[test] fn ple_refusals_leave_the_state() { - let hash = Hash::ple_from_gguf(&ple_fixture(), "qwen4exp", 248_320).unwrap(); + let hash = Hash::ple_from_keys(&ple_fixture().meta, "qwen4exp", 248_320).unwrap(); let mut hist = History::new(3, 248_044); hash.ple_rows_into(&mut hist, 0, &[7, 8], &mut [0; 32]) .unwrap(); @@ -1208,7 +1214,7 @@ mod tests { /// read as EOS differ). #[test] fn ple_media_rows_are_llamacpps() { - let hash = Hash::ple_from_gguf(&ple_fixture(), "qwen4exp", 248_320).unwrap(); + let hash = Hash::ple_from_keys(&ple_fixture().meta, "qwen4exp", 248_320).unwrap(); let want = llamacpp_rows(&hash, &MEDIA_IDS, 248_044); let mut as_eos = MEDIA_IDS; as_eos[3..8].fill(248_044); @@ -1240,7 +1246,7 @@ mod tests { /// same history, and the same refusal of the image id. #[test] fn ple_media_without_spans_is_text_only() { - let hash = Hash::ple_from_gguf(&ple_fixture(), "qwen4exp", 248_320).unwrap(); + let hash = Hash::ple_from_keys(&ple_fixture().meta, "qwen4exp", 248_320).unwrap(); let mut a = hash.new_history().unwrap(); let mut b = hash.new_history().unwrap(); let mut rows_a = vec![0u32; 9 * 16]; @@ -1262,7 +1268,7 @@ mod tests { /// end, between two spans, and past the last. #[test] fn ple_media_refuses_an_image_id_outside_its_span() { - let hash = Hash::ple_from_gguf(&ple_fixture(), "qwen4exp", 248_320).unwrap(); + let hash = Hash::ple_from_keys(&ple_fixture().meta, "qwen4exp", 248_320).unwrap(); let mut hist = hash.new_history().unwrap(); let before = hist.clone(); let cases: [([u32; 6], Vec>); 4] = [ @@ -1292,7 +1298,7 @@ mod tests { /// hash with no image id. #[test] fn ple_media_refuses_malformed_spans() { - let hash = Hash::ple_from_gguf(&ple_fixture(), "qwen4exp", 248_320).unwrap(); + let hash = Hash::ple_from_keys(&ple_fixture().meta, "qwen4exp", 248_320).unwrap(); let mut hist = hash.new_history().unwrap(); let before = hist.clone(); let tokens = [IMG, IMG, IMG, 5]; @@ -1326,7 +1332,7 @@ mod tests { let mut inv = ple_fixture(); inv.meta.retain(|(k, _)| k != "qwen4exp.ple.image_token_id"); - let no_image = Hash::ple_from_gguf(&inv, "qwen4exp", 248_320).unwrap(); + let no_image = Hash::ple_from_keys(&inv.meta, "qwen4exp", 248_320).unwrap(); assert_eq!( no_image.ple_rows_media(&mut hist, 0, &tokens, &one(0..3), &mut [0; 4 * 16]), Err(NgramError::NoImageToken) @@ -1340,7 +1346,7 @@ mod tests { #[test] #[should_panic(expected = "Hash::pad_id: this hash reads raw ids")] fn ple_hash_has_no_pad() { - let hash = Hash::ple_from_gguf(&ple_fixture(), "qwen4exp", 248_320).unwrap(); + let hash = Hash::ple_from_keys(&ple_fixture().meta, "qwen4exp", 248_320).unwrap(); let _ = hash.pad_id(); } @@ -1355,7 +1361,7 @@ mod tests { if let Some(v) = v { inv.meta.push((key.to_string(), v)); } - Hash::ple_from_gguf(&inv, "qwen4exp", 248_320).err() + Hash::ple_from_keys(&inv.meta, "qwen4exp", 248_320).err() }; let key = |e: Option| match e { Some(EngramError::MissingKey { key } | EngramError::KeyValue { key, .. }) => key, diff --git a/crates/engram/src/lib.rs b/crates/engram/src/lib.rs index b2ae15886..db9555831 100644 --- a/crates/engram/src/lib.rs +++ b/crates/engram/src/lib.rs @@ -185,6 +185,16 @@ pub enum EngramError { product: u64, nbytes: u64, }, + /// A table held as runs ([`rows::RunTable`]) of no rows, of empty rows, or of more rows + /// than a `u32` row id names. + #[error( + "{name}: {rows} rows of {row_bytes} B: a run table holds 1 to 2^32 rows of at least one byte" + )] + RunRows { + name: String, + rows: u64, + row_bytes: u64, + }, #[error("{name}: data at {base}+{off} + {nbytes} B runs past the {len} B file {path}")] PastEndOfFile { name: String, @@ -670,7 +680,8 @@ impl TableBytes<'_> { // mapping this type covers is a read-only file mapping (a site's own, // and a split's, whose shards `gguf::Split::open` maps from their // files and nothing else), where it drops the page-table entries and - // the next touch reads the file's bytes again. + // the next touch reads the file's bytes again. A run table's + // (`rows::RunTable`) maps its files read-only too. let rc = unsafe { libc::madvise(first as *mut libc::c_void, len, advice) }; if rc == 0 { Ok(()) diff --git a/crates/engram/src/rows.rs b/crates/engram/src/rows.rs index da2c547d4..3456ef901 100644 --- a/crates/engram/src/rows.rs +++ b/crates/engram/src/rows.rs @@ -14,6 +14,7 @@ //! advised, populated, classified, copied and evicted by expert id. use std::fs::File; +use std::ops::Range; use std::path::{Path, PathBuf}; use std::sync::{Arc, OnceLock}; @@ -292,3 +293,253 @@ impl RowTables for SplitTable { (i == 0).then_some(self as &dyn RowTable) } } + +/// A row-gathered table held as runs of whole rows in files of its own, in row order (a +/// checkpoint folder's PLE table, each part a tensor of the folder's shards), read from the +/// NVMe tier as [`SplitTable`] reads a split's table: one read-only mapping a file, and the +/// same advice, residency query and copy over each run's bytes. +pub struct RunTable { + name: String, + /// Each file's path and mapping, in the order the runs first name them. + files: Vec<(PathBuf, memmap2::Mmap)>, + /// Each run's file, its first byte in that file, and the table's first row past it. + runs: Vec<(usize, usize, u64)>, + row_bytes: u64, + page: usize, +} + +impl RunTable { + /// The table `name` of `row_bytes`-byte rows held as `runs`, in row order: each a file and + /// the byte range of whole rows it holds there. Each file is mapped once, read-only, under + /// `advice`. A table of no rows, of empty rows or past `u32` row ids, a run that is not whole + /// rows and a run past its file's end are refused by name before any advice. + pub fn open( + name: &str, + row_bytes: u64, + runs: &[(PathBuf, Range)], + advice: PageAdvice, + ) -> Result { + let rows_err = |rows: u64| EngramError::RunRows { + name: name.to_owned(), + rows, + row_bytes, + }; + if row_bytes == 0 { + return Err(rows_err(0)); + } + let mut files: Vec<(PathBuf, memmap2::Mmap)> = Vec::new(); + let mut placed = Vec::with_capacity(runs.len()); + let mut end = 0u64; + for (path, at) in runs { + let file = match files.iter().position(|(p, _)| p == path) { + Some(file) => file, + None => { + let io = |op, source| EngramError::SiteIo { + name: name.to_owned(), + path: path.clone(), + op, + source, + }; + let f = File::open(path).map_err(|e| io("open", e))?; + // SAFETY: read-only, and nothing maps the file writable: the contract + // `gguf::Gguf::open_backed` accepts for a split's shards. + let map = unsafe { memmap2::Mmap::map(&f) }.map_err(|e| io("mmap", e))?; + files.push((path.clone(), map)); + files.len() - 1 + } + }; + let len = files[file].1.len() as u64; + let nbytes = at.end.saturating_sub(at.start); + if at.end > len || at.start > at.end { + return Err(EngramError::PastEndOfFile { + name: name.to_owned(), + path: path.clone(), + base: 0, + off: at.start, + nbytes, + len, + }); + } + if !nbytes.is_multiple_of(row_bytes) { + return Err(EngramError::Untiled { + name: name.to_owned(), + rows: nbytes / row_bytes, + row_bytes, + product: nbytes / row_bytes * row_bytes, + nbytes, + }); + } + // Inside a mapping of the file, so inside usize. + let start = at.start as usize; + end += nbytes / row_bytes; + placed.push((file, start, end)); + } + if end == 0 || u32::try_from(end - 1).is_err() { + return Err(rows_err(end)); + } + let table = RunTable { + name: name.to_owned(), + files, + runs: placed, + row_bytes, + page: page_size(), + }; + let (flag, op) = advice.madvise(); + for i in 0..table.runs.len() { + table.view(i).advise_all(flag, op)?; + } + Ok(table) + } + + /// Rows in the table. + pub fn rows(&self) -> u64 { + self.runs.last().map_or(0, |r| r.2) + } + + /// Run `i`'s bytes as the row reads see them. + fn view(&self, i: usize) -> TableBytes<'_> { + let (file, start, end) = self.runs[i]; + let first = i.checked_sub(1).map_or(0, |p| self.runs[p].2); + let len = ((end - first) * self.row_bytes) as usize; + let (path, map) = &self.files[file]; + TableBytes { + name: &self.name, + path, + bytes: &map[start..start + len], + row_bytes: self.row_bytes as usize, + page: self.page, + } + } + + /// `ids` cut into stretches that lie in one run, in order: each run, the stretch's ids as + /// that run's own rows, and where the stretch starts in `ids`. An id past the table is + /// refused by name. + fn stretches(&self, ids: &[u32]) -> Result, usize)>, EngramError> { + let mut out: Vec<(usize, Vec, usize)> = Vec::new(); + for (at, &id) in ids.iter().enumerate() { + let run = self.runs.partition_point(|r| r.2 <= u64::from(id)); + if run == self.runs.len() { + return Err(EngramError::RowOutOfRange { + name: self.name.clone(), + id, + rows: self.rows(), + }); + } + let first = run.checked_sub(1).map_or(0, |p| self.runs[p].2); + // Below the run's end, which `open` held inside u32. + let own = (u64::from(id) - first) as u32; + match out.last_mut() { + Some((r, own_ids, _)) if *r == run => own_ids.push(own), + _ => out.push((run, vec![own], at)), + } + } + Ok(out) + } + + /// `op` over each stretch of `ids` in its run's bytes. + fn each( + &self, + ids: &[u32], + mut op: impl FnMut(TableBytes<'_>, &[u32]) -> Result<(), EngramError>, + ) -> Result<(), EngramError> { + for (run, own, _) in self.stretches(ids)? { + op(self.view(run), &own)?; + } + Ok(()) + } +} + +impl RowTable for RunTable { + fn row_bytes(&self) -> u64 { + self.row_bytes + } + + fn prefetch(&self, ids: &[u32]) -> Result<(), EngramError> { + self.each(ids, |v, own| v.prefetch(own)) + } + + fn populate(&self, ids: &[u32]) -> Result<(), EngramError> { + self.each(ids, |v, own| v.populate(own)) + } + + fn resident_rows(&self, ids: &[u32]) -> Result { + let mut n = 0; + self.each(ids, |v, own| { + n += v.resident_rows(own)?; + Ok(()) + })?; + Ok(n) + } + + fn copy_rows(&self, ids: &[u32], out: &mut [u8]) -> Result<(), EngramError> { + let rb = self.row_bytes as usize; + if out.len() != ids.len() * rb { + return Err(EngramError::OutBufferSize { + name: self.name.clone(), + rows: ids.len(), + want: ids.len() * rb, + got: out.len(), + }); + } + for (run, own, at) in self.stretches(ids)? { + let span = at * rb..(at + own.len()) * rb; + self.view(run).copy_rows(&own, &mut out[span])?; + } + Ok(()) + } +} + +impl RowTables for RunTable { + fn count(&self) -> usize { + 1 + } + + fn table(&self, i: usize) -> Option<&dyn RowTable> { + (i == 0).then_some(self as &dyn RowTable) + } +} + +#[cfg(test)] +mod tests { + use super::{PageAdvice, RunTable}; + use crate::{EngramError, RowTable}; + + /// Rows read across runs in two files come back in the ids' order, each run's own; an id + /// past the table, a run past its file and a run of part of a row are refused by name. + #[test] + fn a_run_table_reads_each_row_from_its_run() { + let dir = std::env::temp_dir().join(format!("engram-runs-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let (a, b) = (dir.join("a"), dir.join("b")); + // Rows of 4 bytes, every byte the row's id; file a holds rows 0..2 after 3 bytes of + // something else, file b rows 2..5 and then rows 5..6 after one row's gap. + let row = |id: u8| [id; 4]; + let mut fa = vec![9u8; 3]; + fa.extend(row(0).iter().chain(&row(1))); + let mut fb: Vec = [2, 3, 4].iter().flat_map(|&i| row(i)).collect(); + fb.extend([7u8; 4].iter().chain(&row(5))); + std::fs::write(&a, &fa).unwrap(); + std::fs::write(&b, &fb).unwrap(); + let run = |path: &std::path::Path, offset: u64, rows: u64| { + (path.to_path_buf(), offset..offset + rows * 4) + }; + let runs = [run(&a, 3, 2), run(&b, 0, 3), run(&b, 16, 1)]; + let t = RunTable::open("t", 4, &runs, PageAdvice::Random).unwrap(); + assert_eq!(t.rows(), 6); + let ids = [5, 0, 1, 2, 4, 3, 1]; + let mut out = vec![0u8; ids.len() * 4]; + t.copy_rows(&ids, &mut out).unwrap(); + let want: Vec = ids.iter().flat_map(|&i| row(i as u8)).collect(); + assert_eq!(out, want); + t.prefetch(&ids).unwrap(); + assert!(matches!( + t.copy_rows(&[6], &mut [0; 4]), + Err(EngramError::RowOutOfRange { id: 6, rows: 6, .. }) + )); + let past = RunTable::open("t", 4, &[run(&b, 16, 2)], PageAdvice::Random); + assert!(matches!(past, Err(EngramError::PastEndOfFile { .. }))); + let torn = RunTable::open("t", 4, &[(a.clone(), 3..9)], PageAdvice::Random); + assert!(matches!(torn, Err(EngramError::Untiled { .. }))); + std::fs::remove_dir_all(&dir).unwrap(); + } +} diff --git a/crates/gguf/src/lib.rs b/crates/gguf/src/lib.rs index e1ec7f8d8..7fc295fee 100644 --- a/crates/gguf/src/lib.rs +++ b/crates/gguf/src/lib.rs @@ -205,8 +205,9 @@ pub enum LoadError { /// A parsed, mmap-backed GGUF file. pub struct Gguf { - // Kept alive for the mapping's lifetime; the mapping borrows the fd. - _file: File, + // Kept alive for the mapping's lifetime; the mapping borrows the fd. A + // header alone ([`Split::header_only`]) has no file. + _file: Option, map: Backing, meta: Vec<(String, Value)>, tensors: Vec, @@ -229,6 +230,8 @@ pub enum Weights { enum Backing { File(Mmap), Anon(MmapMut), + /// The header's bytes and no tensor's ([`Split::header_only`]). + Header(Vec), } impl std::ops::Deref for Backing { @@ -237,6 +240,7 @@ impl std::ops::Deref for Backing { match self { Backing::File(m) => m, Backing::Anon(m) => m, + Backing::Header(h) => h, } } } @@ -322,7 +326,7 @@ impl Gguf { } }; Ok(Gguf { - _file: file, + _file: Some(file), map, meta: h.meta, tensors, @@ -346,6 +350,11 @@ impl Gguf { self.meta.iter().map(|(k, v)| (k.as_str(), v)) } + /// Every metadata pair, in file order. + pub fn keys(&self) -> &[(String, Value)] { + &self.meta + } + /// Raw metadata lookup by exact key. pub fn value(&self, key: &str) -> Option<&Value> { self.meta.iter().find(|(k, _)| k == key).map(|(_, v)| v) @@ -580,6 +589,27 @@ impl Split { Split::assemble(shards, paths) } + /// The header `layout` lays out, as one shard named `path`, holding no tensor's bytes: the + /// model of a checkpoint folder whose tensors a load reads through its part source + /// (`model::part::Source`) while its plan, tokenizer and chat surface read this header as + /// they read a split's. Every tensor is checked as a shard's is (its type, block alignment + /// and size, against the file `layout` would write), and every read of a tensor's bytes is + /// refused by name ([`Gguf::data`]: past the data the shard holds). + pub fn header_only(path: PathBuf, layout: &write::Layout) -> Result { + let bytes = layout.header_bytes().to_vec(); + let h = parse_header(&bytes, layout.file_len())?; + let tensors = strict_tensors(h.tensors, h.data_base, layout.file_len(), &[], true)?; + let shard = Gguf { + _file: None, + map: Backing::Header(bytes), + meta: h.meta, + tensors, + data_base: h.data_base, + alignment: h.alignment, + }; + Split::assemble(vec![shard], vec![path]) + } + /// Index every tensor by name, refusing a name that appears twice. fn assemble(shards: Vec, paths: Vec) -> Result { let mut by_name: HashMap = HashMap::new(); @@ -646,6 +676,11 @@ impl Split { self.shards[0].iter_kv() } + /// [`Gguf::keys`] of the first shard. + pub fn keys(&self) -> &[(String, Value)] { + self.shards[0].keys() + } + /// [`Gguf::architecture`] of the first shard. pub fn architecture(&self) -> Option<&str> { self.shards[0].architecture() @@ -1363,6 +1398,40 @@ mod split_tests { std::fs::remove_dir_all(&d).unwrap(); } + /// A layout's header alone reads as the file the layout writes reads, and holds no tensor + /// bytes: every read of one is refused by name. + #[test] + fn a_header_alone_reads_as_its_file_and_refuses_every_byte() { + let d = set_dir("header"); + let p = d.join("whole.gguf"); + let kvs = [kv(GENERAL_ARCHITECTURE, Value::String("test".into()))]; + write_gguf(&p, &kvs, &["a", "b"]); + let decl = |name: &str| TensorDecl { + name: name.into(), + dims: vec![4], + type_id: 0, + nbytes: 16, + }; + let layout = crate::write::Layout::new(&kvs, vec![decl("a"), decl("b")]).unwrap(); + let file = Split::open(&p).unwrap(); + let head = Split::header_only(d.clone(), &layout).unwrap(); + assert_eq!(head.keys(), file.keys()); + let infos = |s: &Split| { + s.iter_tensors() + .map(|(i, t)| (i, t.name.clone(), t.dims.clone(), t.ty, t.offset, t.nbytes)) + .collect::>() + }; + assert_eq!(infos(&head), infos(&file)); + assert_eq!(head.shard_path(0), Some(d.as_path())); + let shard = head.shard(0).unwrap(); + assert!(!shard.file_backed()); + for (_, t) in head.iter_tensors() { + let e = shard.data(t).unwrap_err().to_string(); + assert!(e.contains(&format!("{:?}", t.name)), "{e}"); + } + std::fs::remove_dir_all(&d).unwrap(); + } + #[test] fn a_missing_shard_is_named() { let d = good_set("missing"); diff --git a/crates/gguf/src/write.rs b/crates/gguf/src/write.rs index 61d6f85ae..b1124d4ac 100644 --- a/crates/gguf/src/write.rs +++ b/crates/gguf/src/write.rs @@ -211,6 +211,11 @@ impl Layout { self.header.len() as u64 } + /// The header's bytes, zero-padded to the data base. + pub fn header_bytes(&self) -> &[u8] { + &self.header + } + /// The alignment every tensor's offset and the data base keep. pub fn alignment(&self) -> u64 { self.alignment diff --git a/crates/gpu-gates/Cargo.toml b/crates/gpu-gates/Cargo.toml index 3e5d0255e..4b959b8be 100644 --- a/crates/gpu-gates/Cargo.toml +++ b/crates/gpu-gates/Cargo.toml @@ -16,12 +16,14 @@ bloomery-gguf = { path = "../gguf" } # node dumps' manifest (re-exported from `lib.rs`), the draft sets, greedy files, KLD bases and # the vision set. bloomery-refset = { path = "../refset" } -# Host only, through the `hf` feature: a server's `--hf [:]`, the repo's GGUF set fetched into a -# local cache, and a DSpark draft's configuration. The gates that read no such thing do not link it. +# Host only, through the `hf` feature: a server's `--hf [:]`, the repo's GGUF set or safetensors +# checkpoint fetched into a local cache, and a DSpark draft's configuration. The gates that read no such thing do +# not link it. bloomery-hf = { path = "../hf", optional = true } # Host-only checkpoint weights, through `hf` for DSpark and `clef` for the decision head. bloomery-safetensors = { path = "../safetensors", optional = true } -# Host only, through the `glm5next-hf` feature: GLM-5.3-Flash's checkpoint map (`generate_glm5next`'s tensor-parallel read). +# Host only, through the `checkpoint` feature: the checkpoint maps, a Qwen3.8 checkpoint folder a server or +# a gate opens (`serve_seats/qwen38_folder.rs`) and GLM-5.3-Flash's (`generate_glm5next`'s tensor-parallel read). bloomery-model-hf = { path = "../model-hf", optional = true } # Host only: the lever registry, which the binaries parse once at `main`. bloomery-levers = { path = "../levers" } @@ -104,8 +106,11 @@ bloomery-vision = { path = "../vision", optional = true } [features] # The hub: `--hf` fetches, plus DSpark configuration and checkpoint weights. hf = ["dep:bloomery-hf", "dep:bloomery-safetensors"] +# A safetensors checkpoint folder read as stored: a server's or a gate's `-m `, and the folder a +# `--hf` repo that holds a checkpoint resolves to. +checkpoint = ["hf", "dep:bloomery-model-hf"] # GLM-5.3-Flash read from its Hugging Face checkpoint folder: `generate_glm5next --place tp`. -glm5next-hf = ["glm5next", "hf", "dep:bloomery-model-hf"] +glm5next-hf = ["glm5next", "checkpoint"] gpu = [ "dep:bloomery-gpu", "dep:bloomery-verbs", diff --git a/crates/gpu-gates/src/bin/bloomery_serve.rs b/crates/gpu-gates/src/bin/bloomery_serve.rs index cf56977be..131493433 100644 --- a/crates/gpu-gates/src/bin/bloomery_serve.rs +++ b/crates/gpu-gates/src/bin/bloomery_serve.rs @@ -324,7 +324,7 @@ models (the --model word is optional with a model file: the file's architecture /// the seat words. A qwen35moe file (Qwen3.6) is the qwen3 seat's. fn seat_of_file(generative: bool) -> Result<(Model, String), GateError> { let path = ref_model_path()?; - let split = Split::open(&path).map_err(|e| format!("open {}: {e}", path.display()))?; + let split = crate::serve_seats::header(&path)?; let (seat, arch) = match split.architecture() { Some(a) if side_seat::row(a).is_some() => { return Err(format!( @@ -391,7 +391,7 @@ models (the --model word is optional with a model file: the file's architecture /// ([`ref_model_path`]) and its `.decision.type`, when it has one. fn file_facts() -> Result<(String, Option), GateError> { let path = ref_model_path()?; - let split = Split::open(&path).map_err(|e| format!("open {}: {e}", path.display()))?; + let split = crate::serve_seats::header(&path)?; let decision = bloomery_gpu_gates::model_file::decision_type(&split)?.map(str::to_owned); Ok(( split.architecture().unwrap_or("").to_owned(), diff --git a/crates/gpu-gates/src/bin/bloomery_serve_qwen38.rs b/crates/gpu-gates/src/bin/bloomery_serve_qwen38.rs index 6cf01ab61..cbe7f6e76 100644 --- a/crates/gpu-gates/src/bin/bloomery_serve_qwen38.rs +++ b/crates/gpu-gates/src/bin/bloomery_serve_qwen38.rs @@ -22,6 +22,11 @@ fn main() -> std::process::ExitCode { #[path = "shared/serve_seats/qwen38.rs"] mod qwen38; +// A Qwen3.8 checkpoint folder, which the seat opens, a sibling of the seat as in `bloomery-serve`. +#[cfg(feature = "gpu")] +#[path = "shared/serve_seats/qwen38_folder.rs"] +mod qwen38_folder; + // The seats' MTP drafts, one a resident slot, a sibling of the seat as in // `bloomery-serve`. #[cfg(feature = "gpu")] @@ -46,7 +51,7 @@ mod tests { use bloomery_levers::Draft38Off; use super::qwen38::{ - ACTS_ON, Feed38, draft_under_media, prompt_cuts, refuse_feed, refuse_lookup_feed, + ACTS_ON, Feed38, draft_off_for, prompt_cuts, refuse_feed, refuse_lookup_feed, refuse_media_feed, }; @@ -163,7 +168,7 @@ mod tests { /// name. #[test] fn a_media_load_refuses_a_set_mtp_draft() { - let why = draft_under_media(Some("mtp"), (true, None)) + let why = draft_off_for(Draft38Off::Media, Some("mtp"), (true, None)) .expect_err("BLOOMERY_DRAFT=mtp beside --mmproj") .to_string(); assert!(why.contains("BLOOMERY_DRAFT=mtp with --mmproj"), "{why}"); @@ -173,7 +178,9 @@ mod tests { /// already off keeps its own reason, and a set `off` is as set. #[test] fn a_media_load_turns_an_unset_draft_off_and_keeps_an_earlier_reason() { - let off = |word, run| draft_under_media(word, run).expect("a word the media load takes"); + let off = |word, run| { + draft_off_for(Draft38Off::Media, word, run).expect("a word the media load takes") + }; assert_eq!(off(None, (true, None)), (false, Some(Draft38Off::Media))); assert_eq!( off(None, (false, Some(Draft38Off::Gate))), @@ -185,6 +192,21 @@ mod tests { ); } + /// A checkpoint folder's load reads no MTP layers: a set `mtp` is refused by name, and an + /// unset draft the rule would have run goes off naming the folder. + #[test] + fn a_folder_load_refuses_a_set_mtp_draft_and_turns_an_unset_one_off() { + let why = draft_off_for(Draft38Off::Folder, Some("mtp"), (true, None)) + .expect_err("BLOOMERY_DRAFT=mtp on a checkpoint folder") + .to_string(); + assert!( + why.contains("BLOOMERY_DRAFT=mtp with a checkpoint folder"), + "{why}" + ); + let off = draft_off_for(Draft38Off::Folder, None, (true, None)).unwrap(); + assert_eq!(off, (false, Some(Draft38Off::Folder))); + } + #[test] fn the_batch_feed_without_a_trace_refuses_nothing() { refuse_feed(BATCH, (2, true), Some("mid-p40-s1")) diff --git a/crates/gpu-gates/src/bin/gate_qwen4exp_nvfp4.rs b/crates/gpu-gates/src/bin/gate_qwen4exp_nvfp4.rs index 5b799dd95..71b18c324 100644 --- a/crates/gpu-gates/src/bin/gate_qwen4exp_nvfp4.rs +++ b/crates/gpu-gates/src/bin/gate_qwen4exp_nvfp4.rs @@ -27,7 +27,8 @@ //! rounding's share of the distance. //! //! Real-only: the fixture holds no NVFP4 stack. Loads the whole file on the card. -//! `BLOOMERY_REF_MODEL` selects a split outside the recipe's model folder. +//! `BLOOMERY_REF_MODEL` selects a split outside the recipe's model folder, or the checkpoint folder +//! itself, which the load reads as stored (`qwen38_folder::open`, a build with the `checkpoint` feature). #[cfg(feature = "gpu")] fn main() -> std::process::ExitCode { @@ -46,6 +47,14 @@ mod e2e; #[path = "shared/q38_oracle.rs"] mod q38_oracle; +#[cfg(feature = "gpu")] +#[path = "shared/serve_seats/qwen38_folder.rs"] +#[allow( + dead_code, + reason = "the gate reads the folder's parts, not its tokenizer" +)] +mod qwen38_folder; + #[cfg(feature = "gpu")] #[path = "shared/qwen38_open.rs"] #[allow( @@ -64,9 +73,10 @@ mod gate { worst_off_path, }; use crate::q38_oracle::{FLIP_ERR_CAP, FREE_BAND, IkRoute, route_flips}; + use crate::qwen38_folder; use crate::qwen38_open::{Open38, open_split}; - use bloomery_gpu::arch::qwen3moe::{Prompt38, Qwen38Model, RouteTap}; + use bloomery_gpu::arch::qwen3moe::{Model38, Prompt38, Qwen38Model, RouteTap}; use bloomery_gpu::host::swap::Residency; use bloomery_gpu::model::StepMode; use bloomery_gpu_gates::flip; @@ -122,9 +132,19 @@ mod gate { Ok(man) } + /// The model at `path`: a split of architecture `qwen4exp` ([`open_split`]), or the checkpoint + /// folder when `path` is a directory ([`qwen38_folder::open`]). + fn model_of(path: &Path) -> Result { + if path.is_dir() { + let f = qwen38_folder::open(path)?; + return Ok((f.parts)(f.header).into()); + } + open_split(path).map(Model38::from) + } + /// (l): the plan on the one visible card, every routed expert on it, and the load. fn open(path: &Path, d: &Dims) -> Result<(Qwen38Model, bool), GateError> { - let file = open_split(path)?; + let file = model_of(path)?; let levers = bloomery_levers::at_main(&[])?; let specs = Place::A.on_host()?.card_specs()?; let [card] = specs[..] else { @@ -377,8 +397,9 @@ mod gate { /// The header's dims, the file's routed gate type, which must be NVFP4, and its dense form /// ([`dense_of`]). fn dims(path: &Path) -> Result { - let file = open_split(path)?; - let inputs = PlanInputs::describe(&file)?; + let model = model_of(path)?; + let file = model.header(); + let inputs = PlanInputs::describe(file)?; let hp = &inputs.hp; let gate = file .find(&names::ffn_gate_exps(0)) diff --git a/crates/gpu-gates/src/bin/shared/qwen38_open.rs b/crates/gpu-gates/src/bin/shared/qwen38_open.rs index 09a5a8425..48887d683 100644 --- a/crates/gpu-gates/src/bin/shared/qwen38_open.rs +++ b/crates/gpu-gates/src/bin/shared/qwen38_open.rs @@ -11,7 +11,7 @@ use std::path::Path; -use bloomery_gpu::arch::qwen3moe::{Body38, Qwen38Model}; +use bloomery_gpu::arch::qwen3moe::{Body38, Model38, Qwen38Model}; use bloomery_gpu::host::swap::Residency; use bloomery_gpu::model::StepMode; use bloomery_gpu_gates::GateError; @@ -89,7 +89,7 @@ pub struct Opened38 { /// ([`Planned38::open`]); the load writes lines of its own to stderr, which a gate's log orders /// after the plan line. pub struct Planned38<'a> { - file: Split, + file: Model38, inputs: PlanInputs, machine: Machine, args: Open38<'a>, @@ -97,10 +97,12 @@ pub struct Planned38<'a> { } impl<'a> Open38<'a> { - /// `file` described and planned. `describe`, not `read`, so the chat surface's two coverage - /// items are not refused here; the load refuses what `ALLOWED` does not name. - pub fn plan(&self, file: Split) -> Result, GateError> { - let mut inputs = PlanInputs::describe(&file)?; + /// `file` (a split, or a checkpoint folder: [`Model38`]) described and planned. `describe`, + /// not `read`, so the chat surface's two coverage items are not refused here; the load + /// refuses what `ALLOWED` does not name. + pub fn plan(&self, file: impl Into) -> Result, GateError> { + let file = file.into(); + let mut inputs = PlanInputs::describe(file.header())?; if let Some(r) = self.room { inputs.room = (r, HostRead::Given); } diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/mod.rs b/crates/gpu-gates/src/bin/shared/serve_seats/mod.rs index 534c71a6c..444022992 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/mod.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/mod.rs @@ -44,6 +44,9 @@ mod served; // as its per-model binary's doc says. #[cfg(feature = "deepseek41")] pub mod qwen38; +// A Qwen3.8 checkpoint folder, which the qwen38 seat opens and [`header`] reads. +#[cfg(feature = "deepseek41")] +pub mod qwen38_folder; // The qwen3 seat sits behind the same server-surface feature. #[cfg(feature = "deepseek41")] pub mod qwen3; @@ -83,3 +86,14 @@ pub mod tiered; // The decide seat: a decision model's backbone on the card, its head on the host. #[cfg(feature = "clef")] pub mod decide; + +/// The header of the model at `path`, which the server seats it by: its GGUF split's first shard, +/// or a checkpoint folder's when `path` is a directory (a Qwen3.8 checkpoint, +/// [`qwen38_folder::open`]). +#[cfg(feature = "deepseek41")] +pub fn header(path: &std::path::Path) -> Result { + if path.is_dir() { + return qwen38_folder::open(path).map(|f| f.header); + } + gguf::Split::open(path).map_err(|e| format!("open {}: {e}", path.display()).into()) +} diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/qwen38.rs b/crates/gpu-gates/src/bin/shared/serve_seats/qwen38.rs index e967038ef..3d1544210 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/qwen38.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/qwen38.rs @@ -18,6 +18,14 @@ //! binaries read the one the profile pins); its first shard gives the //! vocabulary, `tokenizer.chat_template` the chat template //! (`--chat-template-file` replaces it) and `general.name` the default alias. +//! A model path that is a directory is a Qwen3.8 checkpoint folder, served as +//! stored (`qwen38_folder::open`): its header (`gguf::Split::header_only`) +//! takes the first shard's place in every rule and plan, its tokenizer is the +//! vocabulary, and the engine loads every card tensor from the folder's parts +//! ([`Folder38`]) and reads its PLE rows where the folder stores them. Such a +//! load holds every tensor on the stage card and runs no residency machine and +//! no MTP draft: unset, the draft goes off naming the folder +//! ([`draft_off_for`]); `BLOOMERY_DRAFT=mtp` is refused by name. //! The `plan` record, then the `load` and `capture` lines of the qwen4exp //! open, go to stderr as `generate_qwen3moe` prints them, then `listening on //! http://` once the model is loaded and the port is bound (`--port 0` @@ -214,7 +222,7 @@ //! own 400. //! //! An image load drafts nothing, the engine's media load refusing a load that carries the MTP -//! draft ([`draft_under_media`]): `BLOOMERY_DRAFT=mtp` beside `--mmproj` is refused by name +//! draft ([`draft_off_for`]): `BLOOMERY_DRAFT=mtp` beside `--mmproj` is refused by name //! before the plan; unset, the draft goes off with a `load draft=off` record that names //! `--mmproj`, and the lever words that name the draft are then refused as they are on any //! server that drafts nothing. `BLOOMERY_PREFILL=steps` (one step an id) and the route trace @@ -344,8 +352,8 @@ use std::time::{Duration, Instant}; use app::mtp::{MtpBody, MtpDraft}; use bloomery_gpu::arch::qwen3moe::ubatch::ubatch_for; use bloomery_gpu::arch::qwen3moe::{ - Body38, Prompt38, Seq38, TargetRows, checkpoint_budget38, checkpoint_price38, - point_row_budget38, seq38_bytes, + Body38, Folder38, Model38, Prompt38, Seq38, TargetRows, checkpoint_budget38, + checkpoint_price38, point_row_budget38, seq38_bytes, }; use bloomery_gpu::checkpoint::saved::SaveLane; use bloomery_gpu::host::census::TierCensus; @@ -885,26 +893,37 @@ fn draft38( } } -/// [`draft38`]'s answer `(mtp, off)` under `--mmproj`: an image load carries no MTP draft (the -/// engine's `GpuModel::enable_media` refuses a load that does), so `BLOOMERY_DRAFT=mtp` is -/// refused by name; the other words keep the plain path, `off` as set, and a draft the rule would -/// have run goes off with [`Draft38Off::Media`], while a reason it was already off for stands. -/// The lever words that name the draft are then refused by the seat's own rule -/// (`refuse_mtp_levers`). -pub(crate) fn draft_under_media( +/// [`draft38`]'s answer `(mtp, off)` on a load that carries no MTP draft, for the reason `why` +/// ([`Draft38Off::Media`]: an image load, whose sequences the engine's `GpuModel::enable_media` +/// refuses a draft beside; [`Draft38Off::Folder`]: a checkpoint folder's, which reads no MTP +/// layers): `BLOOMERY_DRAFT=mtp` is refused by name; the other words keep the plain path, `off` +/// as set, and a draft the rule would have run goes off with `why`, while a reason it was +/// already off for stands. The lever words that name the draft are then refused by the seat's +/// own rule (`refuse_mtp_levers`). +pub(crate) fn draft_off_for( + why: Draft38Off, word: Option<&str>, (mtp, off): (bool, Option), ) -> Result<(bool, Option), GateError> { if word == Some("mtp") { - return Err( - "BLOOMERY_DRAFT=mtp with --mmproj: an image load carries no MTP draft (the media \ - load refuses it); drop --mmproj, or set BLOOMERY_DRAFT=off (unset, the draft goes \ - off by itself)" - .into(), - ); + let load = match why { + Draft38Off::Folder => { + "with a checkpoint folder: the folder's load reads no MTP layers; \ + serve a prepared split with its draft" + } + _ => { + "with --mmproj: an image load carries no MTP draft (the media load refuses it); \ + drop --mmproj" + } + }; + return Err(format!( + "BLOOMERY_DRAFT=mtp {load}, or set BLOOMERY_DRAFT=off (unset, the draft goes off by \ + itself)" + ) + .into()); } Ok(if mtp { - (false, Some(Draft38Off::Media)) + (false, Some(why)) } else { (false, off) }) @@ -1269,7 +1288,15 @@ pub fn run(args: &[String], side: &side_seat::Flags) -> Result Result draft_off_for(Draft38Off::Folder, levers.draft(), (mtp, draft_off))?, + None => (mtp, draft_off), + }; // A draft lever set on a server that drafts nothing is refused, with // why — asked once the yield below has said its last word, so a draft // the plan turns off refuses them the same as one the rule did. @@ -1338,11 +1369,14 @@ pub fn run(args: &[String], side: &side_seat::Flags) -> Result tok, + None => Tokenizer::from_gguf(&path)?, + }; // `--mmproj`'s encoder file, held against the text model and the vocabulary before anything // is planned. The `--hf` repo's projector is taken only where the load can serve images and // keep what it runs without them: the draft rule's first word is off and the prompt is not - // fed a step an id. An image load takes no MTP draft ([`draft_under_media`]). + // fed a step an id. An image load takes no MTP draft ([`draft_off_for`]). let token = |id: u32| tok.text(id).map(str::to_owned); let vision = vision_of( &a.vision, @@ -1352,19 +1386,17 @@ pub fn run(args: &[String], side: &side_seat::Flags) -> Result Result Result, /// `BLOOMERY_DRAFT=mtp`, the MTP draft; unset the plain path. mtp: bool, /// `BLOOMERY_DRAFT=lookup`, the lookup draft (`drafted::Lookups`). @@ -2086,12 +2123,17 @@ impl Q38 { } let t = Instant::now(); let media = a.vision.is_some(); - let file = Split::open(&a.path).map_err(|e| format!("open {}: {e}", a.path.display()))?; + let model: Model38 = match a.folder { + Some(folder) => folder.into(), + None => Split::open(&a.path) + .map_err(|e| format!("open {}: {e}", a.path.display()))? + .into(), + }; // The room is the main thread's: a reading of the host's available // bytes of its own would plan the load from another room than the // plan the seat ruled by, and the paged tier's arena is that room's // difference. - let mut inputs = PlanInputs::describe(&file)?; + let mut inputs = PlanInputs::describe(model.header())?; inputs.room = a.room.clone(); let cap = serve_ctx(u64::try_from(a.ctx)?, &inputs.hp)?; let ub = ubatch_for(a.ctx)?; @@ -2115,7 +2157,7 @@ impl Q38 { a.plan.hold(&PlanKey::of(&plan)?)?; opened = t.elapsed(); Body38::open_placed_residency( - file, + model, &plan, &inputs, CARD38, @@ -2132,7 +2174,7 @@ impl Q38 { .as_ref() .ok_or("a drafted load with no head picked")?; let draft = open_draft(&a.draft_path, a.draft_from)?; - let mtp = MtpInputs::read(&draft, &file, &inputs, head.rows.clone())?; + let mtp = MtpInputs::read(&draft, model.header(), &inputs, head.rows.clone())?; let machine = a .place .machine(&inputs, ctx_ub, a.experts, Some(&mtp), a.slots)?; @@ -2148,7 +2190,7 @@ impl Q38 { a.plan.hold(&PlanKey::of(&plan.plan)?)?; opened = t.elapsed(); Body38::open_placed_mtp_residency( - file, + model, &plan, &inputs, CARD38, diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/qwen38_folder.rs b/crates/gpu-gates/src/bin/shared/serve_seats/qwen38_folder.rs new file mode 100644 index 000000000..58d75be88 --- /dev/null +++ b/crates/gpu-gates/src/bin/shared/serve_seats/qwen38_folder.rs @@ -0,0 +1,42 @@ +//! A Qwen3.8 checkpoint folder as a server or a gate opens it: a model path that is a directory is +//! one, read as stored (`model_hf::arch::qwen35moe::folder::open`, no prepare step). A build +//! without the `checkpoint` feature refuses it by name. + +use std::path::Path; + +use bloomery_gpu::arch::qwen3moe::Folder38; +use bloomery_gpu_gates::GateError; +use gguf::Split; +use tokenizer::Tokenizer; + +/// A checkpoint folder opened for a load: its header, which plans as a split's first shard does, +/// its tokenizer, and the load of its parts, which takes the header back once it has planned. +pub struct Opened { + pub header: Split, + pub tokenizer: Tokenizer, + pub parts: Box Folder38>, +} + +/// The checkpoint folder `path` opened for a load: a Qwen3.8 checkpoint; any other folder is +/// refused by name. +#[cfg(feature = "checkpoint")] +pub fn open(path: &Path) -> Result { + let f = model_hf::arch::qwen35moe::folder::open(path) + .map_err(|e| format!("checkpoint folder {}: {e}", path.display()))?; + let (source, ple) = (f.source, f.ple); + Ok(Opened { + header: f.header, + tokenizer: f.tokenizer, + parts: Box::new(move |header| Folder38::new(header, source, ple)), + }) +} + +/// A checkpoint folder in a build without the `checkpoint` feature, refused by name. +#[cfg(not(feature = "checkpoint"))] +pub fn open(path: &Path) -> Result { + Err(format!( + "{} is a directory: a checkpoint folder is opened by a build with the checkpoint feature", + path.display() + ) + .into()) +} diff --git a/crates/gpu-gates/src/model_file.rs b/crates/gpu-gates/src/model_file.rs index 60e9132e2..ccb689c2c 100644 --- a/crates/gpu-gates/src/model_file.rs +++ b/crates/gpu-gates/src/model_file.rs @@ -7,7 +7,9 @@ //! (`hf::source`'s), a decision model's head from the repo a `--hf` repo's model card says it //! quantizes (`quantized_from`), a family's MTP draft fetched beside a set only when its //! architecture's rule says the set can run that file, and a seat's image projector under `--hf` -//! (`hf_projector`), the repo's own beside the model. +//! (`hf_projector`), the repo's own beside the model. A `--hf` repo that holds a safetensors +//! checkpoint and no GGUF set resolves to the checkpoint's folder, which the seat that serves it +//! opens. use std::path::PathBuf; use std::sync::OnceLock; @@ -24,8 +26,8 @@ pub use fetch::{ }; /// The refusal when nothing names the model. -pub(crate) const NONE: &str = "no model file: pass -m PATH (the first shard) or --hf [:], \ - or set BLOOMERY_REF_MODEL"; +pub(crate) const NONE: &str = "no model file: pass -m PATH (the first shard, or a checkpoint folder) \ + or --hf [:], or set BLOOMERY_REF_MODEL"; /// A present decision marker is a string; malformed metadata is not an absent head. pub fn decision_type(file: &Split) -> Result, GateError> { diff --git a/crates/gpu-gates/src/model_file/fetch.rs b/crates/gpu-gates/src/model_file/fetch.rs index a88f537e6..810fc1024 100644 --- a/crates/gpu-gates/src/model_file/fetch.rs +++ b/crates/gpu-gates/src/model_file/fetch.rs @@ -46,6 +46,11 @@ fn print(e: &Event<'_>) { .u("files", set.files.len()) .u("bytes", set.bytes()) .eprint(), + Event::Checkpoint { repo, files, bytes } => Record::new(&record::HF_CHECKPOINT) + .w("repo", repo) + .u("files", files) + .u("bytes", bytes) + .eprint(), Event::File { file, bytes, diff --git a/crates/gpu-gates/src/record.rs b/crates/gpu-gates/src/record.rs index cee7dfb47..d5efaf410 100644 --- a/crates/gpu-gates/src/record.rs +++ b/crates/gpu-gates/src/record.rs @@ -2086,6 +2086,18 @@ pub static HF_SET: Kind = Kind { ], }; +/// The safetensors checkpoint a `--hf` repo with no GGUF set holds. +pub static HF_CHECKPOINT: Kind = Kind { + name: "hf_checkpoint", + head: "hf checkpoint", + doc: "The safetensors checkpoint `--hf ` resolved to, the repo holding no GGUF set: the checkpoint folder is served as stored, and its files and their bytes are fetched.", + parts: &[ + key("repo", Word, ""), + key("files", U64, ""), + key("bytes", U64, "B"), + ], +}; + /// A file of a `--hf` fetch before it runs. pub static HF_FILE: Kind = Kind { name: "hf_file", @@ -2579,7 +2591,7 @@ pub mod groups { /// `model_file/fetch.rs`. pub mod fetch { use crate::record::{ - HF_DONE, HF_DRAFT_SKIP, HF_FILE, HF_OFFLINE, HF_PROGRESS, HF_SET, Kind, + HF_CHECKPOINT, HF_DONE, HF_DRAFT_SKIP, HF_FILE, HF_OFFLINE, HF_PROGRESS, HF_SET, Kind, }; /// A fetch of exact files: the lines `resolve_exact` prints, which @@ -2587,8 +2599,9 @@ pub mod groups { pub const FETCH: &[&Kind] = &[&HF_OFFLINE, &HF_FILE, &HF_PROGRESS, &HF_DONE]; /// A `--hf` set's resolve (`resolve`, through `model_file::name`), - /// which prints its own two besides the fetch's. - pub const SET: &[&Kind] = &[&HF_SET, &HF_DRAFT_SKIP]; + /// which prints its own three besides the fetch's: a GGUF set's or a + /// checkpoint's, and a skipped draft. + pub const SET: &[&Kind] = &[&HF_SET, &HF_CHECKPOINT, &HF_DRAFT_SKIP]; } /// `bin/shared/serve_seats/drafted.rs`. diff --git a/crates/gpu-glm5next/src/tp/mod.rs b/crates/gpu-glm5next/src/tp/mod.rs index e2c1941c1..0f12759b5 100644 --- a/crates/gpu-glm5next/src/tp/mod.rs +++ b/crates/gpu-glm5next/src/tp/mod.rs @@ -31,7 +31,7 @@ use std::sync::Arc; use bloomery_gpu::head::Head; use bloomery_gpu::host::{GatherLoop, GatherSet, GatherShape, Gathers}; use bloomery_gpu::hybrid::SlotMap; -use bloomery_gpu::weights::{TensorPart, Weights, block_index, part_format}; +use bloomery_gpu::weights::{Weights, block_index}; use bloomery_gpu::{Branch, DeviceTensor, Gpu, GpuError, MappedHost, WindowMut, mem_batch}; use bloomery_gpu_deepseek41::hc::HC_STREAMS; use bloomery_gpu_deepseek41::router::glm5next::{N_EXPERT, N_USED}; @@ -39,7 +39,7 @@ use cuda_core::{CudaStream, DeviceBuffer, sys}; use gguf::Split; use model::arch::glm5next::names::Sub; use model::arch::glm5next::place::{KdaLanes, PlanInputs}; -use model::part::{Part, Source, Take}; +use model::part::{Source, Take}; use runtime::layer::{FfnKind, MixerKind}; use crate::body::pair::write_row; @@ -613,15 +613,11 @@ impl Drop for Rank { } } -/// The host bytes of a source's parts uploaded together: a load holds at most this much of its -/// own beside the card, past one tensor's. A file's parts are its mapping's and count none. -const GROUP: usize = 1 << 30; - /// Rank `at`'s parts of the trunk's `layers` layers of `heads` heads and of the head, from -/// `source` on `gpu`: every tensor but the token embedding, which the host reads, and the layers -/// past the trunk's, each the part the family's cut gives the rank ([`RankOf::take`]). Every part -/// is planned and its card format checked ([`part_format`]), and refused by name, before any is -/// read or uploaded; the parts go up a group at a time ([`GROUP`]). +/// `source` on `gpu` ([`bloomery_gpu::parts::load`]): every tensor the card holds +/// (`model::arch::glm5next::tp::card_holds`: not the token embedding, which the host reads, nor +/// the layers past the trunk's), each the part the family's cut gives the rank +/// ([`RankOf::take`]), in its type's card format. pub fn load( gpu: &Gpu, source: &impl Source, @@ -629,45 +625,18 @@ pub fn load( layers: usize, heads: usize, ) -> Result { - let planned = source - .plan(|name, ty, dims| { + bloomery_gpu::parts::load( + gpu.stream(), + source, + |name, ty, dims| { if !model::arch::glm5next::tp::card_holds(name, block_index(name)?, layers) { return Ok(Take::Skip); } at.take(name, ty, dims, heads as u64) - }) - .map_err(|e| shape(e.to_string()))?; - for p in &planned { - part_format(&p.info)?; - } - let mut w = Weights::default(); - let mut group: Vec> = Vec::new(); - let mut held = 0; - for p in &planned { - let part = source.read(p).map_err(|e| shape(e.to_string()))?; - held += part.held(); - group.push(part); - if held >= GROUP { - upload(gpu, &mut w, &group)?; - group.clear(); - held = 0; - } - } - upload(gpu, &mut w, &group)?; - Ok(w) -} - -/// Upload `group` beside the parts `w` holds. -fn upload(gpu: &Gpu, w: &mut Weights, group: &[Part<'_>]) -> Result<(), GpuError> { - let parts: Vec> = group - .iter() - .map(|p| TensorPart { - info: p.info.clone(), - spans: p.spans(), - scale: p.scale.clone(), - }) - .collect(); - w.add_parts(gpu.stream(), &parts) + }, + |_| None, + false, + ) } /// `gpu`'s context current on this thread, which runs every rank of a loopback in turn, and its diff --git a/crates/gpu/src/arch/qwen3moe/body38.rs b/crates/gpu/src/arch/qwen3moe/body38.rs index 60177442a..86d8de7b0 100644 --- a/crates/gpu/src/arch/qwen3moe/body38.rs +++ b/crates/gpu/src/arch/qwen3moe/body38.rs @@ -156,7 +156,8 @@ use crate::host::{BatchLeg, PassKind, StepLeg, refuse_tier_count}; use crate::hybrid::{Boundary, BoundaryShape, Chain, HostResidency, Hybrid, SlotMap}; use crate::media::{ImageSpan, MediaBody, MediaSpan, RopeBase, SeqRope, kept_rule, rope_table}; use crate::model::{ - ChainBody, GpuModel, HostServed, Rollback, RowHeads, Rows, SlotRange, SlotRows, Slots, + CardBytes, ChainBody, GpuModel, HostServed, Rollback, RowHeads, Rows, SlotRange, SlotRows, + Slots, }; use crate::prompt_timing::{PromptStats, PromptTiming, nanos}; use crate::rope_table::{RopeSpec, RopeTable}; @@ -165,13 +166,14 @@ use crate::weights::{Weights, tensor_scales}; use crate::{DeviceTensor, Gpu, GpuError, launch_u32, ple}; use bloomery_levers::{ChurnHosting, HostCfg}; use cuda_core::{CudaStream, DeviceBuffer}; -use engram::Hash; use engram::hash::{History, Window}; use engram::prefetch::{FillMode, HelperOptions, Prefetcher, caller_sibling}; -use engram::rows::SplitTable; +use engram::rows::{PageAdvice, RunTable, SplitTable}; +use engram::{Hash, RowTables}; use gguf::quant::{E4m3Rows, Nvfp4Rows, dequant_row}; use gguf::{GgmlType, Split, TensorInfo}; use model::arch::Arch; +use model::part::{Source, StoredRows}; use model::placement::{Device, Plan}; use runtime::seqstate::{Kept, Price, Take}; use runtime::swaprule::KeptRows; @@ -493,14 +495,28 @@ struct PleBatch { } impl PleRead { - /// The reader of `file`'s table `name`, for fills of up to `max_rows` - /// rows; its helper on the SMT sibling of the loading thread's core when - /// that thread is pinned (the step thread loads), else floating. - fn open(file: &Arc, name: &str, max_rows: usize) -> Result { - let table = - SplitTable::open(Arc::clone(file), name).map_err(|e| GpuError::plan(WHAT, e))?; + /// The reader of `file`'s table `name`, or of its rows where `stored` + /// holds them (a checkpoint folder's, [`RunTable`]), for fills of up to + /// `max_rows` rows; its helper on the SMT sibling of the loading thread's + /// core when that thread is pinned (the step thread loads), else + /// floating. + fn open( + file: &Arc, + stored: Option<&StoredRows>, + name: &str, + max_rows: usize, + ) -> Result { + let table: Arc = match stored { + None => Arc::new( + SplitTable::open(Arc::clone(file), name).map_err(|e| GpuError::plan(WHAT, e))?, + ), + Some(s) => Arc::new( + RunTable::open(name, s.row_bytes, &s.runs, PageAdvice::Random) + .map_err(|e| GpuError::plan(WHAT, e))?, + ), + }; let rows = Prefetcher::over( - Arc::new(table), + table, &[max_rows], HelperOptions { mode: FillMode::Touch, @@ -590,9 +606,12 @@ impl PleHost { /// shard's keys, for calls of up to /// `cap` positions, read from `tier` — the plan's /// ([`Plan::row_tier`]): the host, or the NVMe tier through its reader - /// ([`PleRead`]); any other tier is refused by name. Load-time only. + /// ([`PleRead`]); any other tier is refused by name. A checkpoint + /// folder's table (`file` its header) is read where `stored` holds its F8 + /// rows, at its scale, from the NVMe tier alone. Load-time only. fn new( file: Arc, + stored: Option<&StoredRows>, n_vocab: usize, cap: usize, tier: Device, @@ -603,14 +622,10 @@ impl PleHost { name: name.clone(), need, }; - let path = file - .shard_path(0) - .ok_or(GpuError::state(WHAT, "the file's first shard"))?; - let meta = gguf::inventory_of(path).map_err(|e| GpuError::plan(WHAT, e))?; let vocab = u32::try_from(n_vocab) .map_err(|_| GpuError::shape(WHAT, format!("a vocabulary of {n_vocab}")))?; - let hash = - Hash::ple_from_gguf(&meta, "qwen4exp", vocab).map_err(|e| GpuError::plan(WHAT, e))?; + let hash = Hash::ple_from_keys(file.keys(), "qwen4exp", vocab) + .map_err(|e| GpuError::plan(WHAT, e))?; let hist = hash .new_history() .map_err(|e| GpuError::shape(WHAT, e.to_string()))?; @@ -621,19 +636,21 @@ impl PleHost { let width = info.dims.first().copied().unwrap_or(0) as usize; let rows = info.dims.get(1).copied().unwrap_or(0) as usize; let per_token = (geo::HIDDEN).checked_div(width).unwrap_or(0); - let scale = || { - tensor_scales(WHAT, &info, |n| { + let scale = || match stored { + Some(s) => Ok(s.scale), + None => tensor_scales(WHAT, &info, |n| { let (s, t) = file.find(n)?; Some((file.shard(s)?, t)) })? .and_then(|s| s.first().copied()) - .ok_or_else(|| refuse("a scale beside its rows")) + .ok_or_else(|| refuse("a scale beside its rows")), }; - let decode = match info.ty { - GgmlType::IQ4_NL => PleRows::Iq4Nl, - GgmlType::F8E4M3_ROWS => PleRows::F8(Box::new(E4m3Rows::new(scale()?))), - GgmlType::NVFP4_ROWS => PleRows::Nvfp4(Box::new(Nvfp4Rows::new(scale()?))), - _ => return Err(refuse("IQ4_NL, F8 or NVFP4 rows")), + let decode = match (info.ty, stored) { + (GgmlType::IQ4_NL, None) => PleRows::Iq4Nl, + (GgmlType::F8E4M3_ROWS, _) => PleRows::F8(Box::new(E4m3Rows::new(scale()?))), + (GgmlType::NVFP4_ROWS, None) => PleRows::Nvfp4(Box::new(Nvfp4Rows::new(scale()?))), + (_, None) => return Err(refuse("IQ4_NL, F8 or NVFP4 rows")), + (_, Some(_)) => return Err(refuse("F8 rows where a checkpoint folder stores them")), }; let (Some(blck), Some(size)) = (info.ty.blck_size(), info.ty.type_size()) else { return Err(refuse("a sized type")); @@ -643,12 +660,20 @@ impl PleHost { return Err(refuse("rows of a whole part of the model width")); } let row_bytes = width / blck * size; - if rows == 0 || info.nbytes != (rows * row_bytes) as u64 { + if rows == 0 + || info.nbytes != (rows * row_bytes) as u64 + || stored.is_some_and(|s| s.row_bytes != row_bytes as u64) + { return Err(refuse("a whole number of rows")); } let nvme = match tier { + Device::Host if stored.is_some() => { + return Err(refuse( + "on the NVMe tier, where a checkpoint folder's rows are read", + )); + } Device::Host => None, - Device::Nvme => Some(PleRead::open(&file, &name, cap * per_token)?), + Device::Nvme => Some(PleRead::open(&file, stored, &name, cap * per_token)?), Device::Card(_) | Device::Unused => { return Err(refuse( "on the host or the NVMe tier, where the plan reads it", @@ -1166,6 +1191,90 @@ pub struct Body38 { who: Identity, } +/// The model a Qwen3.8 load reads ([`Body38::open_placed`] and its siblings): a GGUF split, or a +/// checkpoint folder ([`Folder38`]). +pub enum Model38 { + Split(Split), + Folder(Folder38), +} + +impl Model38 { + /// The header the load plans by: the split's first shard's, or the folder's + /// (`gguf::Split::header_only`). + pub fn header(&self) -> &Split { + match self { + Model38::Split(file) => file, + Model38::Folder(f) => &f.header, + } + } +} + +impl From for Model38 { + fn from(file: Split) -> Model38 { + Model38::Split(file) + } +} + +impl From for Model38 { + fn from(folder: Folder38) -> Model38 { + Model38::Folder(folder) + } +} + +/// The card's tensors of a plan from a part source ([`crate::parts::load_placed`]). +type CardParts = + Box, usize, bool) -> Result + Send>; + +/// A checkpoint folder as a Qwen3.8 load reads it: its header (`gguf::Split::header_only`, the +/// header its plan was made from), its card tensors through its part source, and the PLE +/// table's rows where the folder stores them, which the table's NVMe reader maps +/// ([`StoredRows`]). The load holds every tensor on its one card and the PLE table on the NVMe +/// tier ([`crate::parts::OnCard`]): no host set, so no residency machine, and the folder's MTP +/// layers are not read, so no draft. Each card part's file pages leave the page cache once it is +/// staged under the host config's `card_dontneed`, as a split's card segments do. +pub struct Folder38 { + header: Split, + parts: CardParts, + ple: Option, +} + +impl Folder38 { + /// The folder of header `header`, its card tensors read from `source` and its PLE rows at + /// `ple`. + pub fn new( + header: Split, + source: S, + ple: Option, + ) -> Folder38 { + Folder38 { + header, + parts: Box::new(move |stream, plan, card, release| { + crate::parts::load_placed(stream, &source, plan, card, release) + }), + ple, + } + } +} + +/// A folder's load under `residency`, `drafted` or not, refused by name unless it runs no +/// residency machine and no draft ([`Folder38`]). +fn refuse_folder(residency: Residency, drafted: bool) -> Result<(), GpuError> { + if residency != Residency::Off { + return Err(GpuError::shape( + WHAT, + "a checkpoint folder's load runs under BLOOMERY_RESIDENCY=off: a residency machine \ + moves experts through a host set, which the folder's load does not hold", + )); + } + if drafted { + return Err(GpuError::shape( + WHAT, + "a checkpoint folder's load runs no MTP draft: the folder's MTP layers are not read", + )); + } + Ok(()) +} + impl Body38 { /// Card `card` of `plan`, which `inputs` made, resident: the plan's /// segments, the joins ([`Body38::derive`]) and the body over them with @@ -1181,9 +1290,11 @@ impl Body38 { /// or not every layer, a /// layer or a width the kernels do not take, a `ubatch` outside /// `1..=min(UBATCH, ctx)`, and a ubatch arena past what the plan's - /// scratch counts. + /// scratch counts. `file` is a GGUF split or a checkpoint folder + /// ([`Model38`]); a folder's load is refused by name under a residency + /// machine or with an MTP draft ([`Folder38`]). pub fn open_placed( - file: Split, + file: impl Into, plan: &Plan<'_>, inputs: &model::arch::qwen35moe::place::PlanInputs, card: usize, @@ -1210,7 +1321,7 @@ impl Body38 { /// and a sequence past the count is refused by name. Zero slots is /// refused by name. pub fn open_placed_slots( - file: Split, + file: impl Into, plan: &Plan<'_>, inputs: &model::arch::qwen35moe::place::PlanInputs, card: usize, @@ -1250,7 +1361,7 @@ impl Body38 { reason = "open_placed's six, the residency and its pool's hosting, and the planned slots (rust-quality R8)" )] pub fn open_placed_residency( - file: Split, + file: impl Into, plan: &Plan<'_>, inputs: &model::arch::qwen35moe::place::PlanInputs, card: usize, @@ -1274,7 +1385,7 @@ impl Body38 { reason = "open_placed's six and the draft's file and inputs (rust-quality R8)" )] pub fn open_placed_mtp( - file: Split, + file: impl Into, plan: &model::arch::qwen35moe::place::MtpPlan<'_>, inputs: &model::arch::qwen35moe::place::PlanInputs, card: usize, @@ -1306,7 +1417,7 @@ impl Body38 { reason = "open_placed_mtp's eight and the slot count (rust-quality R8)" )] pub fn open_placed_mtp_slots( - file: Split, + file: impl Into, plan: &model::arch::qwen35moe::place::MtpPlan<'_>, inputs: &model::arch::qwen35moe::place::PlanInputs, card: usize, @@ -1339,7 +1450,7 @@ impl Body38 { reason = "open_placed_mtp's eight, the residency and its pool's hosting, and the planned slots (rust-quality R8)" )] pub fn open_placed_mtp_residency( - file: Split, + file: impl Into, plan: &model::arch::qwen35moe::place::MtpPlan<'_>, inputs: &model::arch::qwen35moe::place::PlanInputs, card: usize, @@ -1375,7 +1486,7 @@ impl Body38 { reason = "open_placed's six, the residency and its pool's hosting, the draft's file, inputs and plan, and the planned slots (rust-quality R8)" )] fn open_with( - file: Split, + file: impl Into, plan: &Plan<'_>, inputs: &model::arch::qwen35moe::place::PlanInputs, card: usize, @@ -1393,6 +1504,17 @@ impl Body38 { if slots == 0 { return Err(GpuError::shape(WHAT, "a load of 0 resident sequence slots")); } + let (file, folder) = match file.into() { + Model38::Split(file) => (file, None), + Model38::Folder(f) => (f.header, Some((f.parts, f.ple))), + }; + if folder.is_some() { + refuse_folder(residency, mtp.is_some())?; + } + let (bytes, stored) = match &folder { + Some((parts, ple)) => (CardBytes::Parts(parts.as_ref()), ple.as_ref()), + None => (CardBytes::File, None), + }; refuse_tier_count(WHAT, plan.machine.tiers.len())?; let refused: Vec = inputs .unimplemented() @@ -1440,12 +1562,14 @@ impl Body38 { max_rows: PASS_ROWS, stacks: Arc::new(swap38::stacks(&inputs.model, plan)?), }; - GpuModel::load_placed_with( + GpuModel::load_placed_from( file, + bytes, plan, card, host, spec, + (&[], 0), |stream, _, layers, w| Body38::derive(stream, &shape.kinds, layers, w), |gpu, file, w, set, glue| { let draft = mtp @@ -1453,7 +1577,7 @@ impl Body38 { .transpose()?; let mut body = Body38::load_placed( gpu, - file, + (file, stored), w, plan, inputs, @@ -1514,11 +1638,11 @@ impl Body38 { /// sized ([`HostServed::start_residency`], the load's last step). #[allow( clippy::too_many_arguments, - reason = "the load's card handle, file, weights, plan, inputs and chain shape, the host tier's residency and levers, the residency glue, and the ubatch with the arena bytes the plan counts and the planned slots (rust-quality R8)" + reason = "the load's card handle, file with its stored PLE rows, weights, plan, inputs and chain shape, the host tier's residency and levers, the residency glue, and the ubatch with the arena bytes the plan counts and the planned slots (rust-quality R8)" )] fn load_placed( gpu: &Gpu, - file: &Arc, + (file, stored): (&Arc, Option<&StoredRows>), w: &Weights, plan: &Plan<'_>, inputs: &model::arch::qwen35moe::place::PlanInputs, @@ -1734,7 +1858,13 @@ impl Body38 { .row_tier() .map_err(|e| GpuError::plan(WHAT, e))? .ok_or(GpuError::state(WHAT, "a plan that places the PLE table"))?; - let ple = PleHost::new(Arc::clone(file), spec.vocab as usize, host_cols, ple_tier)?; + let ple = PleHost::new( + Arc::clone(file), + stored, + spec.vocab as usize, + host_cols, + ple_tier, + )?; let ckpt = Checkpoints::sequence( gpu.context(), checkpoint_lens_of(&stores), @@ -5537,8 +5667,9 @@ const _: () = assert!(VERIFY_ROWS <= PASS_ROWS); #[cfg(test)] mod tests { use super::{ - Arc, Device, E4m3Rows, GgmlType, GpuError, Nvfp4Rows, PleHost, Split, checkpoint_budget38, - checkpoint_bytes, checkpoint_lens, checkpoint_price38, dequant_row, ple_takes, + Arc, Device, E4m3Rows, GgmlType, GpuError, Nvfp4Rows, PleHost, Residency, Split, + checkpoint_budget38, checkpoint_bytes, checkpoint_lens, checkpoint_price38, dequant_row, + ple_takes, refuse_folder, }; use engram::hash::{EosWindow, Window}; @@ -5599,13 +5730,11 @@ mod tests { assert!(e.to_string().contains("token 0:"), "{e}"); } - /// A one-file split of the PLE keys (16 heads of 5 rows, 160 values a - /// row) and the table `rows` of type id `ty`, with `scale` as its - /// `.scale` when given, opened as the body opens its file, and the PLE host over it. The - /// file is removed once the host is open. - fn ple_host(ty: u32, rows: &[u8], scale: Option, cap: usize) -> Result { + /// The header of a one-file split of the PLE keys (16 heads of 5 rows, 160 values a row) + /// and the table of type id `ty` in `nbytes`, with its `.scale` when `scaled`. + fn ple_layout(ty: u32, nbytes: usize, scaled: bool) -> gguf::write::Layout { use gguf::Value; - use gguf::write::{Layout, TensorDecl, Writer}; + use gguf::write::{Layout, TensorDecl}; let k = |s: &str| format!("qwen4exp.ple.{s}"); let u64s = |v: Vec| Value::Array(v.into_iter().map(Value::U64).collect()); let kvs = [ @@ -5627,26 +5756,45 @@ mod tests { type_id, nbytes: nbytes as u64, }; - let scale = scale.map(f32::to_le_bytes); - let mut tensors = vec![( - decl("per_layer_token_embd.weight", vec![160, 80], ty, rows.len()), - rows, + let mut tensors = vec![decl( + "per_layer_token_embd.weight", + vec![160, 80], + ty, + nbytes, )]; - if let Some(b) = &scale { - tensors.push(( - decl("per_layer_token_embd.scale", vec![1], 0, 4), - b.as_slice(), - )); + if scaled { + tensors.push(decl("per_layer_token_embd.scale", vec![1], 0, 4)); } - let layout = Layout::new(&kvs, tensors.iter().map(|t| t.0.clone()).collect()).unwrap(); - let path = - std::env::temp_dir().join(format!("body38-ple-{ty}-{}.gguf", std::process::id())); + Layout::new(&kvs, tensors).unwrap() + } + + /// The split of [`ple_layout`] holding the table `rows` and `scale` as its `.scale` when + /// given, opened as the body opens its file, and the PLE host over it. The file, its own + /// whichever test runs beside it, is removed once the host is open. + fn ple_host(ty: u32, rows: &[u8], scale: Option, cap: usize) -> Result { + use gguf::write::Writer; + use std::sync::atomic::{AtomicUsize, Ordering}; + static FILES: AtomicUsize = AtomicUsize::new(0); + let layout = ple_layout(ty, rows.len(), scale.is_some()); + let path = std::env::temp_dir().join(format!( + "body38-ple-{ty}-{}-{}.gguf", + std::process::id(), + FILES.fetch_add(1, Ordering::Relaxed) + )); let mut w = Writer::new(std::fs::File::create(&path).unwrap(), layout).unwrap(); - for (d, b) in &tensors { - w.tensor(&d.name, b).unwrap(); + w.tensor("per_layer_token_embd.weight", rows).unwrap(); + if let Some(scale) = scale { + w.tensor("per_layer_token_embd.scale", &scale.to_le_bytes()) + .unwrap(); } w.finish().unwrap(); - let host = PleHost::new(Arc::new(Split::open(&path).unwrap()), 16, cap, Device::Host); + let host = PleHost::new( + Arc::new(Split::open(&path).unwrap()), + None, + 16, + cap, + Device::Host, + ); std::fs::remove_file(&path).unwrap(); host } @@ -5716,4 +5864,77 @@ mod tests { ); } } + + /// A checkpoint folder's load runs no residency machine and no draft: either is refused by + /// name, and a load under neither passes. + #[test] + fn a_folder_load_refuses_residency_and_a_draft() { + assert!(refuse_folder(Residency::Off, false).is_ok()); + let mid = Residency::Mid { + pinned: 1, + spares: 1, + }; + for (residency, drafted, need) in [ + (mid, false, "BLOOMERY_RESIDENCY=off"), + (Residency::Off, true, "no MTP draft"), + ] { + let e = refuse_folder(residency, drafted).unwrap_err().to_string(); + assert!(e.contains(need), "{e}"); + } + } + + /// A checkpoint folder's F8 table, its rows in runs of two files where the folder stores + /// them and its header alone in the split, decodes from the NVMe tier's reader the values + /// the split holding the same rows decodes on the host; on the host, and as rows of another + /// type, the folder's table is refused by name. + #[test] + fn a_folder_s_stored_rows_decode_as_the_split_s() { + let f8: Vec = (0..80 * 160u32) + .map(|i| (i * 53 % 256) as u8) + .map(|b| if b & 0x7F == 0x7F { 0x38 } else { b }) + .collect(); + let tokens = [7u32, 3, 9, 2, 1, 5]; + let mut want = ple_host(GgmlType::F8E4M3_ROWS.as_u32(), &f8, Some(0.375), 8).unwrap(); + want.fill(0, &tokens).unwrap(); + let dir = std::env::temp_dir().join(format!("body38-ple-folder-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let (a, b) = (dir.join("a.safetensors"), dir.join("b.safetensors")); + let cut = 30 * 160; + std::fs::write(&a, [&[9u8; 7][..], &f8[..cut]].concat()).unwrap(); + std::fs::write(&b, &f8[cut..]).unwrap(); + let stored = model::part::StoredRows { + row_bytes: 160, + runs: vec![(a, 7..7 + cut as u64), (b, 0..(f8.len() - cut) as u64)], + scale: 0.375, + }; + let open = |ty: GgmlType, tier| { + let row = if ty == GgmlType::F8E4M3_ROWS { 160 } else { 90 }; + let layout = ple_layout(ty.as_u32(), 80 * row, true); + let header = Split::header_only(dir.clone(), &layout).unwrap(); + PleHost::new(Arc::new(header), Some(&stored), 16, 8, tier) + }; + let mut got = open(GgmlType::F8E4M3_ROWS, Device::Nvme).unwrap(); + got.fill(0, &tokens).unwrap(); + assert_eq!(got.ids[..got.named], want.ids[..want.named]); + assert!( + got.e.len() == want.e.len() + && got + .e + .iter() + .zip(&want.e) + .all(|(g, w)| g.to_bits() == w.to_bits()) + ); + for (ty, tier, need) in [ + (GgmlType::F8E4M3_ROWS, Device::Host, "on the NVMe tier"), + ( + GgmlType::NVFP4_ROWS, + Device::Nvme, + "F8 rows where a checkpoint", + ), + ] { + let e = open(ty, tier).err().expect("refused"); + assert!(e.to_string().contains(need), "{ty}: {e}"); + } + std::fs::remove_dir_all(&dir).unwrap(); + } } diff --git a/crates/gpu/src/arch/qwen3moe/mod.rs b/crates/gpu/src/arch/qwen3moe/mod.rs index bfddf8a05..2449dc5f6 100644 --- a/crates/gpu/src/arch/qwen3moe/mod.rs +++ b/crates/gpu/src/arch/qwen3moe/mod.rs @@ -57,9 +57,9 @@ pub use body35::{ Body35, DecodeInput35, LayerKind35, Open35, checkpoint_budget35, checkpoint_price35, }; pub use body38::{ - ALLOWED, Body38, CHECKPOINT_EVERY, DecodeInput38, LayerKind38, Prompt38, Qwen38Model, RouteTap, - Seq38, Store38Host, XSTREAM_COSTS, checkpoint_budget38, checkpoint_bytes, checkpoint_price38, - point_row_budget38, seq_positional_bytes, seq38_bytes, + ALLOWED, Body38, CHECKPOINT_EVERY, DecodeInput38, Folder38, LayerKind38, Model38, Prompt38, + Qwen38Model, RouteTap, Seq38, Store38Host, XSTREAM_COSTS, checkpoint_budget38, + checkpoint_bytes, checkpoint_price38, point_row_budget38, seq_positional_bytes, seq38_bytes, }; pub use draft_taps::TapRowsSink; pub use mtp35::{Mtp35, addresses}; diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index aaa857323..920ece0e4 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -69,6 +69,7 @@ pub mod moe_fused; pub mod mtp; pub mod mxfp4; pub mod mxfp4_sel; +pub mod parts; pub mod ple; pub mod probe; pub mod prompt_timing; diff --git a/crates/gpu/src/model.rs b/crates/gpu/src/model.rs index 198176005..d03229286 100644 --- a/crates/gpu/src/model.rs +++ b/crates/gpu/src/model.rs @@ -695,7 +695,16 @@ impl GpuModel { derive: impl FnOnce(&CudaStream, &Split, Range, &mut Weights) -> Result<(), GpuError>, body: impl FnOnce(&Gpu, Split, &Weights, HostResidency) -> Result, ) -> Result, GpuError> { - let (p, residency) = placed_pre(&file, plan, card, host, (&[], 0), &[], derive)?; + let (p, residency) = placed_pre( + &file, + CardBytes::File, + plan, + card, + host, + (&[], 0), + &[], + derive, + )?; let t_body = Instant::now(); let body = body(&p.gpu, file, &p.weights, residency)?; placed_done(p, body, t_body.elapsed()) @@ -756,6 +765,43 @@ impl GpuModel { HostResidency, ResidencyGlue, ) -> Result, + ) -> Result, GpuError> { + GpuModel::load_placed_from( + file, + CardBytes::File, + plan, + card, + host, + residency, + hosted, + derive, + body, + ) + } + + /// [`GpuModel::load_placed_hosting`] with the card's tensors read from `bytes`: the file's + /// own, or a part source's when `file` is a checkpoint folder's header + /// ([`CardBytes::Parts`]). + #[allow( + clippy::too_many_arguments, + reason = "load_placed_hosting's arguments and where the card's bytes come from (rust-quality R8)" + )] + pub fn load_placed_from( + file: Split, + bytes: CardBytes<'_>, + plan: &Plan<'_>, + card: usize, + host: HostCfg, + residency: ResidencySpec, + hosted: (&[(usize, ExpertList)], u64), + derive: impl FnOnce(&CudaStream, &Split, Range, &mut Weights) -> Result<(), GpuError>, + body: impl FnOnce( + &Gpu, + &Arc, + &Weights, + HostResidency, + ResidencyGlue, + ) -> Result, ) -> Result, GpuError> { const WHAT: &str = "GpuModel::load_placed_with"; let churn = match residency.lever { @@ -787,7 +833,7 @@ impl GpuModel { (&joined[..], pool.1 + hosted.1) }; let file = Arc::new(file); - let (p, set) = placed_pre(&file, plan, card, host, extra, known, derive)?; + let (p, set) = placed_pre(&file, bytes, plan, card, host, extra, known, derive)?; let t_body = Instant::now(); let glue = match residency.lever { Residency::Off => ResidencyGlue::off(), @@ -2054,9 +2100,20 @@ where Ok(hybrid) } +/// Where a placed load's card tensors come from ([`GpuModel::load_placed_from`]). +pub enum CardBytes<'a> { + /// The file's own mapped tensors ([`Weights::load_placed`]). + File, + /// A part source's, for a file that is a checkpoint folder's header + /// (`gguf::Split::header_only`): card `card` of `plan` loaded on the stream, each part's file + /// pages released when the host config's `card_dontneed` asks (`crate::parts::load_placed`). + Parts(&'a dyn Fn(&CudaStream, &Plan<'_>, usize, bool) -> Result), +} + /// The steps of a placed load before its body: the plan's card and context /// budget, the card, the host's memory against the plan's host need, the -/// card's weights with `derive`'s filings, and — after the uploads, so the +/// card's weights (`bytes`': the file's, or a part source's) with `derive`'s +/// filings, and — after the uploads, so the /// card's file bytes have left the page cache first — the plan's host set /// read in and locked as `host` asks, with `extra`'s runs (of `extra`'s /// bytes) beside it and `known`'s runs known, neither read in nor locked nor @@ -2066,8 +2123,13 @@ where /// (`BLOOMERY_NEED_HOOK`, first, before the load's own open of the card), a plan's card /// that is not one visible device ([`Gpu::open_card`]), and a /// host whose `MemAvailable` is under the plan's need ([`HostNeed`]). +#[allow( + clippy::too_many_arguments, + reason = "the load's file and where its card bytes come from, its plan and card, the host's levers, the extra and known host runs, and the derive (rust-quality R8)" +)] fn placed_pre( file: &Split, + bytes: CardBytes<'_>, plan: &Plan<'_>, card: usize, host: HostCfg, @@ -2108,7 +2170,12 @@ fn placed_pre( .grant(available, least) .map_err(|e| GpuError::plan(what, format!("{e}{}", plan.host.room_clause())))?; let t = Instant::now(); - let mut weights = Weights::load_placed(gpu.stream(), file, plan, card, host.card_dontneed)?; + let mut weights = match bytes { + CardBytes::File => { + Weights::load_placed(gpu.stream(), file, plan, card, host.card_dontneed)? + } + CardBytes::Parts(load) => load(gpu.stream(), plan, card, host.card_dontneed)?, + }; let upload = t.elapsed(); let t = Instant::now(); derive(gpu.stream(), file, layers.clone(), &mut weights)?; diff --git a/crates/gpu/src/parts.rs b/crates/gpu/src/parts.rs new file mode 100644 index 000000000..7400b6fcf --- /dev/null +++ b/crates/gpu/src/parts.rs @@ -0,0 +1,295 @@ +//! The one load of a part source onto a card (`model::part::Source`: a GGUF file or a +//! checkpoint folder). The caller's rule says which tensors the card holds, whole or the part +//! of a rank, as the source plans them; every planned part's card format is checked before any +//! is read; then each part is read and staged in turn through one staging ring, which one +//! synchronize ends ([`load`]). A rank of a tensor-parallel load of N ranks passes its family's +//! cut; a one-card load passes its placement plan's card ([`OnCard`]). + +use std::collections::{BTreeMap, HashMap, HashSet}; + +use cuda_core::CudaStream; +use gguf::{GgmlType, TensorInfo}; +use model::part::{Source, Take}; +use model::placement::{CardFormat, Device, Format, ModelTensor, Plan, Segment}; + +use crate::GpuError; +use crate::upload::UploadRing; +use crate::weights::{TensorPart, Weights, part_format, stage_parts}; + +/// What the errors name. +const WHAT: &str = "parts::load"; + +/// The parts of `source` that `rule` takes, each read and uploaded to the card of `stream` in +/// the card format `card` names for it, else its type's ([`part_format`]). Every part is planned +/// and its format checked, and refused by name, before any is read; then one part at a time is +/// read and staged, so the load holds one part's bytes of its own beside the card (a file's +/// parts are its mapping's and hold none). The weights are the caller's only once every copy +/// has ended: a refusal at any part drops what was staged. Under `release` each part's file +/// pages are dropped once it is staged ([`Source::release`]), as a split's placed load drops its +/// card segments' (`BLOOMERY_CARD_DONTNEED`). +pub fn load( + stream: &CudaStream, + source: &S, + rule: impl Fn(&str, GgmlType, &[u64]) -> Result, + card: impl Fn(&TensorInfo) -> Option, + release: bool, +) -> Result { + let planned = source.plan(rule).map_err(|e| refused(e.to_string()))?; + let mut budget = 0usize; + for p in &planned { + budget = budget + .checked_add(part_format(&p.info)?.1) + .ok_or_else(|| refused("the parts' staging bytes pass usize".into()))?; + } + stream.context().bind_to_thread()?; + let mut staged = BTreeMap::new(); + // Made after `staged` and dropped before it: its drop drains the copies, so no buffer of a + // refused load is freed while a copy still writes it. + let mut ring = UploadRing::new(stream, budget)?; + for p in &planned { + let part = source.read(p).map_err(|e| refused(e.to_string()))?; + let one = TensorPart { + info: part.info.clone(), + spans: part.spans(), + scale: part.scale.clone(), + }; + let held = |name: &str| staged.contains_key(name); + let added = stage_parts(stream, &mut ring, &[one], held, &card)?; + staged.extend(added); + drop(part); + if release { + source.release(p).map_err(|e| refused(e.to_string()))?; + } + } + ring.finish(stream)?; + Ok(Weights::of_staged(staged)) +} + +/// Card `card` of `plan` as a load from a part source reads it ([`load`]): each tensor the plan +/// holds whole on that card, in the plan's card format and resident bytes; a tensor the plan +/// uses nowhere, or reads by rows from the NVMe tier (a row-gathered table, which its own +/// reader maps where the source stores it), is skipped. A source fills no host set and no other +/// card, so a tensor the plan splits or holds on the host or another card is refused by name. +pub struct OnCard { + held: HashMap, + skipped: HashSet, +} + +impl OnCard { + /// Card `card` of `plan`, each tensor's segments checked as the type says. + pub fn of(plan: &Plan<'_>, card: usize) -> Result { + let model = plan.model; + let rows = plan.rows.iter().map(|row| { + model + .tensors + .get(row.tensor) + .map(|t| (t, row.segments.as_slice(), model.experts_of(t))) + .ok_or_else(|| refused(format!("a placement row names tensor {}", row.tensor))) + }); + OnCard::of_rows(rows, card) + } + + /// The tensors of `rows` (each with its segments and its stack's experts) on card `card`. + fn of_rows<'a>( + rows: impl Iterator>, + card: usize, + ) -> Result { + let mut held = HashMap::new(); + let mut skipped = HashSet::new(); + for row in rows { + let (t, segments, experts) = row?; + let elsewhere = |placed: String| { + refused(format!( + "{} is placed {placed}: a load from a part source holds every tensor whole \ + on card {card}, and reads a row-gathered table from the NVMe tier", + t.name + )) + }; + match segments { + [] => { + skipped.insert(t.name.clone()); + } + [s] if s.device == Device::Card(card) => { + let Format::Card(format) = s.format else { + return Err(refused(format!( + "{} is on the card as {}", + t.name, s.format + ))); + }; + let whole = s + .experts + .as_ref() + .is_none_or(|e| e.ids().len() as u64 == experts); + if !whole { + return Err(elsewhere(format!( + "on card {card} in part, the rest of its experts elsewhere" + ))); + } + held.insert(t.name.clone(), (format, s.resident_bytes)); + } + [s] if s.device == Device::Unused + || (s.device == Device::Nvme && t.gathered_rows.is_some()) => + { + skipped.insert(t.name.clone()); + } + segments => { + let on: Vec = + segments.iter().map(|s| format!("{:?}", s.device)).collect(); + return Err(elsewhere(format!("on {}", on.join(" and ")))); + } + } + } + Ok(OnCard { held, skipped }) + } + + /// The rule [`load`] plans by: a tensor the card holds whole, or none; a name the plan does + /// not hold is refused by name. + pub fn take(&self, name: &str) -> Result { + if self.held.contains_key(name) { + Ok(Take::Whole) + } else if self.skipped.contains(name) { + Ok(Take::Skip) + } else { + Err(format!("{name} is not a tensor of the plan")) + } + } + + /// The plan's card format of `t`. + pub fn format(&self, t: &TensorInfo) -> Option { + self.held.get(&t.name).map(|&(f, _)| f) + } + + /// Refused by name unless `w` holds every tensor the card holds, at the plan's resident + /// bytes: a tensor the source does not make, or makes otherwise than the plan's file did. + pub fn check(&self, w: &Weights) -> Result<(), GpuError> { + for (name, &(_, bytes)) in &self.held { + let got = w.get(name).map(|dw| dw.resident_bytes() as u64); + if got != Some(bytes) { + return Err(refused(format!( + "{name}: the source uploaded {got:?} device bytes, the plan holds {bytes}" + ))); + } + } + Ok(()) + } +} + +/// Card `card` of `plan` from `source` ([`OnCard`], [`load`], each part's pages released under +/// `release`), checked against the plan. +pub fn load_placed( + stream: &CudaStream, + source: &S, + plan: &Plan<'_>, + card: usize, + release: bool, +) -> Result { + let on = OnCard::of(plan, card)?; + let w = load( + stream, + source, + |name, _, _| on.take(name), + |t| on.format(t), + release, + )?; + on.check(&w)?; + Ok(w) +} + +fn refused(detail: String) -> GpuError { + GpuError::shape(WHAT, detail) +} + +#[cfg(test)] +mod tests { + use super::*; + use model::placement::{ExpertList, Role}; + + /// The rule a load from a part source plans by: a tensor whole on the card is taken in the + /// plan's card format, an unused tensor and a row-gathered table on the NVMe tier are + /// skipped, and a tensor the plan places on the host, on another card, split, or in part + /// on the card is refused by name, as is a name the plan does not hold; the check refuses + /// weights that miss a held tensor. + #[test] + fn on_card_takes_the_card_s_tensors_and_refuses_the_rest() { + let tensor = |name: &str, role, gathered_rows| ModelTensor { + name: name.into(), + shard: 0, + layer: Some(0), + role, + ty: GgmlType::BF16, + dims: vec![64, 4, 8], + file_bytes: 4096, + gathered_rows, + }; + let seg = |device, format, experts: Option>| Segment { + device, + format, + experts: experts.map(|ids| ExpertList::new(ids, 8).unwrap()), + resident_bytes: 4096, + }; + let bf16 = Format::Card(CardFormat::Bf16Raw); + let held = tensor("a.weight", Role::Attention, None); + let stack = tensor("x.weight", Role::RoutedExperts, None); + let table = tensor("t.weight", Role::EngramTable, Some(1)); + let unused = tensor("u.weight", Role::Attention, None); + let ok = [ + (&held, vec![seg(Device::Card(0), bf16, None)]), + ( + &stack, + vec![seg(Device::Card(0), bf16, Some((0..8).collect()))], + ), + (&table, vec![seg(Device::Nvme, Format::NvmeFile, None)]), + (&unused, vec![seg(Device::Unused, Format::Unused, None)]), + ]; + let of = |r: &[(&ModelTensor, Vec)]| { + OnCard::of_rows(r.iter().map(|(t, s)| Ok((*t, s.as_slice(), 8))), 0) + }; + let on = of(&ok).unwrap(); + assert!(matches!(on.take("a.weight"), Ok(Take::Whole))); + assert!(matches!(on.take("x.weight"), Ok(Take::Whole))); + assert!(matches!(on.take("t.weight"), Ok(Take::Skip))); + assert!(matches!(on.take("u.weight"), Ok(Take::Skip))); + assert!(on.take("z.weight").unwrap_err().contains("z.weight")); + let info = |name: &str| TensorInfo { + name: name.into(), + dims: vec![64], + ty: GgmlType::BF16, + offset: 0, + nbytes: 128, + }; + assert_eq!(on.format(&info("a.weight")), Some(CardFormat::Bf16Raw)); + assert_eq!(on.format(&info("t.weight")), None); + let e = on.check(&Weights::default()).unwrap_err().to_string(); + assert!(e.contains("uploaded None device bytes"), "{e}"); + let host_table = tensor("t.weight", Role::EngramTable, Some(1)); + for (t, segments, named) in [ + ( + &held, + vec![seg(Device::Host, Format::HostFile, None)], + "on Host", + ), + (&held, vec![seg(Device::Card(1), bf16, None)], "on Card(1)"), + ( + &host_table, + vec![seg(Device::Host, Format::HostFile, None)], + "on Host", + ), + ( + &held, + vec![ + seg(Device::Card(0), bf16, None), + seg(Device::Host, Format::HostFile, None), + ], + "on Card(0) and Host", + ), + ( + &stack, + vec![seg(Device::Card(0), bf16, Some((0..6).collect()))], + "in part", + ), + ] { + let e = of(&[(t, segments)]).err().expect("refused").to_string(); + assert!(e.contains(&t.name) && e.contains(named), "{e}"); + } + } +} diff --git a/crates/gpu/src/weights.rs b/crates/gpu/src/weights.rs index 655826ce0..a02789c6f 100644 --- a/crates/gpu/src/weights.rs +++ b/crates/gpu/src/weights.rs @@ -329,6 +329,36 @@ fn part_formats( Ok((formats, budget)) } +/// Upload `parts` through `ring`, whose copies the caller ends with one synchronize, each part +/// in the card format `card` names for it, else its type's ([`part_format`]): every part checked +/// first as [`Weights::add_parts`] checks them, a name `held` holds refused. The uploaded weights +/// by name, for the caller to file once its copies end. +pub(crate) fn stage_parts( + stream: &CudaStream, + ring: &mut UploadRing, + parts: &[TensorPart<'_>], + held: impl Fn(&str) -> bool, + card: impl Fn(&TensorInfo) -> Option, +) -> Result, GpuError> { + let (formats, _) = part_formats(parts, held)?; + let mut added = Vec::with_capacity(parts.len()); + for (p, format) in parts.iter().zip(formats) { + let t = &p.info; + let dw = upload_rows( + stream, + ring, + PARTS, + card(t).unwrap_or(format), + t, + tensor_rows(t), + &p.spans, + p.scale.clone(), + )?; + added.push((t.name.clone(), dw)); + } + Ok(added) +} + /// The resident weights of a layer range: every `blk.L.*` tensor with L in /// `layers`, what the architecture derives for those blocks, and — when /// loaded with globals — the non-block tensors. Load-time only; per-step @@ -466,29 +496,24 @@ impl Weights { stream: &CudaStream, parts: &[TensorPart<'_>], ) -> Result<(), GpuError> { - let (formats, budget) = part_formats(parts, |name| self.by_name.contains_key(name))?; + let held = |name: &str| self.by_name.contains_key(name); + let (_, budget) = part_formats(parts, held)?; stream.context().bind_to_thread()?; let mut ring = UploadRing::new(stream, budget)?; - let mut added = Vec::with_capacity(parts.len()); - for (p, format) in parts.iter().zip(formats) { - let t = &p.info; - let dw = upload_rows( - stream, - &mut ring, - PARTS, - format, - t, - tensor_rows(t), - &p.spans, - p.scale.clone(), - )?; - added.push((t.name.clone(), dw)); - } + let added = stage_parts(stream, &mut ring, parts, held, |_| None)?; ring.finish(stream)?; self.by_name.extend(added); Ok(()) } + /// The weights a load staged ([`stage_parts`]), once its ring's copies have ended. + pub(crate) fn of_staged(by_name: BTreeMap) -> Weights { + Weights { + by_name, + tied: false, + } + } + /// [`Weights::load_where`], and with `tied` the token embedding is also the head /// ([`Weights::get`]). fn load_named( diff --git a/crates/hf/src/fetch.rs b/crates/hf/src/fetch.rs index 7fad8004e..9dfb03395 100644 --- a/crates/hf/src/fetch.rs +++ b/crates/hf/src/fetch.rs @@ -91,6 +91,13 @@ impl State { pub enum Event<'a> { /// The set a quant picked. Set { repo: &'a str, set: &'a GgufSet }, + /// The safetensors checkpoint a repo with no GGUF set holds (`crate::checkpoint_files`): + /// its files and their bytes. + Checkpoint { + repo: &'a str, + files: usize, + bytes: u64, + }, /// A file before its fetch: its size, its state, and the byte the /// fetch starts at (its size when cached). File { @@ -592,7 +599,9 @@ impl Client { } /// The set `r` picks in its repo, every file fetched and checked; the - /// local paths in shard order (the first shard first). `draft`, the MTP + /// local paths in shard order (the first shard first). A repo with no + /// GGUF set and a safetensors checkpoint ([`crate::checkpoint_files`]) + /// gives the checkpoint's folder, its files fetched and checked. `draft`, the MTP /// draft a model family runs beside its set, is fetched too when the /// repo holds a file of its name ([`crate::mtp_draft`]) and the fetched /// set can run it ([`Draft::usable`]; a `DraftSkip` event says why not): @@ -626,6 +635,15 @@ impl Client { Listed::Cache(why) => return self.offline_set(r, &why, events), }; let sets = gguf_sets(&r.repo, &entries)?; + if sets.is_empty() + && let Some(files) = crate::checkpoint_files(&entries) + { + let dir = checkpoint(r, &self.root, &files, events)?; + for e in &files { + self.fetch(&r.repo, e, events)?; + } + return Ok(vec![dir]); + } let set = crate::pick(&r.repo, &sets, r.quant.as_deref())?; events(&Event::Set { repo: &r.repo, set }); let mut files = Vec::with_capacity(set.files.len()); @@ -679,6 +697,14 @@ impl Client { }; let entries = verified_entries(&self.root, &r.repo).map_err(offline)?; let sets = gguf_sets(&r.repo, &entries).map_err(offline)?; + if sets.is_empty() + && let Some(files) = crate::checkpoint_files(&entries) + { + events(&Event::Offline { repo: &r.repo, why }); + let dir = checkpoint(r, &self.root, &files, events).map_err(offline)?; + self.offline_files(&r.repo, &files, events)?; + return Ok(vec![dir]); + } let set = crate::pick(&r.repo, &sets, r.quant.as_deref()).map_err(offline)?; events(&Event::Offline { repo: &r.repo, why }); events(&Event::Set { repo: &r.repo, set }); @@ -739,6 +765,30 @@ impl Client { } } +/// The folder in `root`'s cache of the checkpoint `files` of `r`'s repo, which a resolve gives +/// in place of a set's first shard, its event sent; a quant is refused by name, as the +/// checkpoint is the repo's one model. +fn checkpoint( + r: &RepoRef, + root: &Path, + files: &[Entry], + events: &mut dyn FnMut(&Event<'_>), +) -> Result { + if let Some(quant) = &r.quant { + return Err(HfError::CheckpointQuant { + repo: r.repo.clone(), + quant: quant.clone(), + }); + } + events(&Event::Checkpoint { + repo: &r.repo, + files: files.len(), + bytes: files.iter().map(|f| f.size).sum(), + }); + let index = cache_path(root, &r.repo, safetensors::Folder::INDEX)?; + Ok(index.parent().unwrap_or(root).to_path_buf()) +} + /// The files under `repo`'s cache directory in `root` that a fetch checked: /// each one beside its `.verified` marker, its path relative to the repo's /// directory, its size on disk and the digest the marker names. A file @@ -950,6 +1000,7 @@ mod tests { Event::DraftSkip { .. } => "draft-skip".to_owned(), Event::Progress { .. } => "progress".to_owned(), Event::Set { .. } => "set".to_owned(), + Event::Checkpoint { .. } => "checkpoint".to_owned(), Event::Offline { .. } => "offline".to_owned(), }) }); @@ -1338,6 +1389,56 @@ mod tests { ); } + /// A repo with no GGUF set and a safetensors checkpoint at its top level resolves to the + /// checkpoint's folder in the cache, its index, shards, configs and template fetched and + /// nothing else; offline the verified checkpoint stands in. A quant on such a repo is + /// refused by name, and a repo with neither model is refused naming both. + #[test] + fn a_safetensors_repo_resolves_to_its_checkpoint_folder() { + let d = scratch("checkpoint"); + let files = [ + "model.safetensors.index.json", + "model-00001-of-00002.safetensors", + "model-00002-of-00002.safetensors", + "config.json", + "tokenizer.json", + "chat_template.jinja", + "README.md", + "extra/other.safetensors", + ]; + let body = content(); + let listed: Vec<(&str, &[u8])> = files.iter().map(|f| (*f, body.as_slice())).collect(); + let c = hub(&d, "a/b", &listed); + let (got, seen) = offline_run(&c, "a/b"); + assert_eq!(got.expect("resolve"), [d.join("cache/a/b")], "{seen:?}"); + let fetched: Vec<&str> = files[..6].to_vec(); + for f in &files { + assert_eq!( + d.join("cache/a/b").join(f).exists(), + fetched.contains(f), + "{f}" + ); + } + let (got, seen) = offline_run(&unreachable(&d.join("cache")), "a/b"); + assert_eq!(got.expect("offline"), [d.join("cache/a/b")], "{seen:?}"); + assert_eq!( + seen.iter() + .filter(|s| s.ends_with("offline-cached")) + .count(), + 6 + ); + match offline_run(&c, "a/b:Q4_K_M").0 { + Err(HfError::CheckpointQuant { quant, .. }) => assert_eq!(quant, "Q4_K_M"), + other => panic!("{other:?}"), + } + let none = hub(&d, "a/c", &[("README.md", b"x")]); + let e = offline_run(&none, "a/c").0.expect_err("no model"); + assert!( + matches!(e, HfError::NoGguf { .. }) && e.to_string().contains("safetensors"), + "{e}" + ); + } + #[test] fn offline_zero_or_two_verified_sets_are_refused() { let d = scratch("offline-zero-two"); diff --git a/crates/hf/src/lib.rs b/crates/hf/src/lib.rs index 53179da25..d164a88fd 100644 --- a/crates/hf/src/lib.rs +++ b/crates/hf/src/lib.rs @@ -79,8 +79,16 @@ pub enum HfError { Listing { repo: String, why: String }, #[error("{repo}: the listing names {path:?}, which is not a relative path inside the repo")] Path { repo: String, path: String }, - #[error("{repo} holds no GGUF model file")] + #[error( + "{repo} holds no GGUF model file and no safetensors checkpoint (a model.safetensors.index.json at its top level)" + )] NoGguf { repo: String }, + /// A quant named for a repo whose model is a safetensors checkpoint: the checkpoint is the + /// repo's one model, in the formats it stores. + #[error( + "{repo} holds a safetensors checkpoint, which is served as stored: --hf {repo} names it; :{quant} picks a GGUF set" + )] + CheckpointQuant { repo: String, quant: String }, #[error("{repo} holds {} GGUF sets and no quant was given; pick one with --hf {repo}:: {}", .sets.len(), .sets.join(", "))] NoQuant { repo: String, sets: Vec }, #[error("{repo}: no GGUF set carries the quant {quant:?}; its quants: {}", .tags.join(", "))] @@ -394,6 +402,30 @@ fn tag_of(name: &str) -> String { base[at..].to_ascii_uppercase() } +/// The safetensors checkpoint `entries` hold at the repo's top level: its index +/// (`model.safetensors.index.json`), every `.safetensors` file, and the configs, tokenizer and +/// chat template beside them (`.json`, `.jinja`), in the listing's order. `None` for a repo with +/// no index there. +pub fn checkpoint_files(entries: &[Entry]) -> Option> { + let top = |e: &&Entry| !e.path.contains('/'); + entries + .iter() + .filter(top) + .any(|e| e.path == safetensors::Folder::INDEX) + .then(|| { + entries + .iter() + .filter(top) + .filter(|e| { + [".safetensors", ".json", ".jinja"] + .iter() + .any(|x| e.path.ends_with(x)) + }) + .cloned() + .collect() + }) +} + /// The GGUF model sets of `entries`, in the listing's order of their first /// file. A split set lacking a shard (or holding one twice) is /// [`HfError::IncompleteSet`]. diff --git a/crates/hf/src/modelopt.rs b/crates/hf/src/modelopt.rs index 5bd93a0d4..ab391ea07 100644 --- a/crates/hf/src/modelopt.rs +++ b/crates/hf/src/modelopt.rs @@ -887,6 +887,51 @@ fn fp8_append<'a>( Ok([rows, k]) } +/// The tensor scale of the FP8 row table `name` placed as `placed` ([`Format::Fp8Tensor`] read as +/// [`Use::Rows`], whose bytes [`normalize`] keeps as stored), and its companion's name, from the +/// headers and that one value: a table that is not 2-D F8_E4M3, or a companion that is not one +/// finite F32 or BF16, is refused by name. A reader that maps the table's rows where they are +/// stored checks its parts here without copying them. +pub fn fp8_rows_scale<'a, 'p>( + get: &impl Fn(&str) -> Result, safetensors::Error>, + name: &str, + placed: &'p Placed, +) -> Result<(f32, &'p str), Error> { + let refuse = |what: String| Error::Format { + name: name.to_string(), + format: placed.format, + what, + }; + let raw = get(name)?; + if placed.format != Format::Fp8Tensor || raw.dtype != Dtype::F8E4M3 || raw.shape.len() != 2 { + return Err(refuse(format!( + "is {} {:?}, not F8_E4M3 rows", + raw.dtype.name(), + raw.shape + ))); + } + let [scale_name] = placed.companions.as_slice() else { + return Err(refuse(format!( + "reads {:?}, not one tensor scale", + placed.companions + ))); + }; + let s = get(scale_name)?; + finite(scale_name, &s)?; + let scale = match (s.dtype, s.bytes) { + (Dtype::F32, &[a, b, c, d]) => f32::from_le_bytes([a, b, c, d]), + (Dtype::Bf16, &[a, b]) => f32::from_bits(u32::from(u16::from_le_bytes([a, b])) << 16), + _ => { + return Err(refuse(format!( + "scale {scale_name} is {} {:?}, not one F32 or BF16", + s.dtype.name(), + s.shape + ))); + } + }; + Ok((scale, scale_name)) +} + /// A tensor of a format that is neither NVFP4 nor stacked: FP8 rows, or BF16 and F32. fn plain<'a>( get: impl Fn(&str) -> Result, safetensors::Error>, @@ -903,32 +948,7 @@ fn plain<'a>( let raw = get(name)?; let dims: Vec = raw.shape.iter().rev().map(|&d| d as u64).collect(); if placed.format == Format::Fp8Tensor { - if raw.dtype != Dtype::F8E4M3 || raw.shape.len() != 2 { - return Err(refuse(format!( - "is {} {:?}, not F8_E4M3 rows", - raw.dtype.name(), - raw.shape - ))); - } - let [scale_name] = placed.companions.as_slice() else { - return Err(refuse(format!( - "reads {:?}, not one tensor scale", - placed.companions - ))); - }; - let s = get(scale_name)?; - finite(scale_name, &s)?; - let scale = match (s.dtype, s.bytes) { - (Dtype::F32, &[a, b, c, d]) => f32::from_le_bytes([a, b, c, d]), - (Dtype::Bf16, &[a, b]) => f32::from_bits(u32::from(u16::from_le_bytes([a, b])) << 16), - _ => { - return Err(refuse(format!( - "scale {scale_name} is {} {:?}, not one F32 or BF16", - s.dtype.name(), - s.shape - ))); - } - }; + let (scale, scale_name) = fp8_rows_scale(&get, name, placed)?; let decoder = E4m3Rows::new(scale); let mut values = [0.0; 256]; for (chunk, bytes) in raw.bytes.chunks(values.len()).enumerate() { diff --git a/crates/levers/src/lib.rs b/crates/levers/src/lib.rs index e5c138e34..ccbfed61c 100644 --- a/crates/levers/src/lib.rs +++ b/crates/levers/src/lib.rs @@ -2047,6 +2047,9 @@ pub enum Draft38Off { /// sequence a rotation table the draft's walks do not read, so it carries /// no draft; a set [`DRAFT`]`=mtp` is refused. Media, + /// Unset on the Qwen3.8 seat serving a checkpoint folder: the folder's + /// load reads no MTP layers; a set [`DRAFT`]`=mtp` is refused. + Folder, } impl fmt::Display for Draft38Off { @@ -2091,6 +2094,10 @@ impl fmt::Display for Draft38Off { "unset: --mmproj makes an image load, whose sequences hold rotation tables the \ MTP draft's walks do not read", ), + Draft38Off::Folder => f.write_str( + "unset: a checkpoint folder's load reads no MTP layers; prepare the folder into a \ + split and its MTP draft to draft", + ), } } } diff --git a/crates/model-hf/src/arch/qwen35moe.rs b/crates/model-hf/src/arch/qwen35moe.rs index ade341f58..0d66b8ae0 100644 --- a/crates/model-hf/src/arch/qwen35moe.rs +++ b/crates/model-hf/src/arch/qwen35moe.rs @@ -23,6 +23,8 @@ use serde_json::Value as Json; use model::arch::qwen35moe::hparams::{MAX_PLE_HEADS, MAX_PLE_NGRAM}; use model::arch::qwen35moe::names::{self, Sub}; +pub mod folder; + /// Why the checkpoint was refused. #[derive(Debug, thiserror::Error)] pub enum Error { diff --git a/crates/model-hf/src/arch/qwen35moe/folder.rs b/crates/model-hf/src/arch/qwen35moe/folder.rs new file mode 100644 index 000000000..466f8931f --- /dev/null +++ b/crates/model-hf/src/arch/qwen35moe/folder.rs @@ -0,0 +1,524 @@ +//! A Qwen3.8 checkpoint folder as the program reads it: the tensors of its split, each declared +//! in the GGUF type its stored format and role give ([`plan_tensors`]), the header those tensors +//! and the folder's keys make ([`read`]), and serving's tokenizer checked against Hugging Face's +//! ([`served_as_hf`]). `prepare` writes that header and the tensors' bytes as a split; a load +//! opens the folder as it is ([`open`]): the same header held in memory +//! (`gguf::Split::header_only`), the tensors a card holds read through the folder's part source +//! ([`crate::part::Checkpoint`]), and the PLE table's rows read where the folder stores them +//! (`model::part::StoredRows`). Both read the folder through these functions alone, so a folder +//! and the split `prepare` writes of it describe one model. + +use std::collections::BTreeMap; +use std::path::Path; + +use bloomery_levers::PREPARE_Q8; +use gguf::write::{Layout, TensorDecl}; +use gguf::{GgmlType, Split, Value}; +use hf::modelopt::{self, Plain, Use}; +use hf::quant_config::{self, Config, Format, Plan}; +use safetensors::{Dtype, Folder}; + +use model::part::StoredRows; + +use super::{Checkpoint, Mapped}; +use crate::part::{self, Made}; + +/// Why a folder was refused. +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error(transparent)] + Map(#[from] super::Error), + #[error(transparent)] + Part(#[from] part::Error), + #[error(transparent)] + Tensors(#[from] safetensors::Error), + #[error(transparent)] + Quant(#[from] quant_config::Error), + #[error(transparent)] + Normal(#[from] modelopt::Error), + #[error(transparent)] + Vocab(#[from] hf::vocab::Error), + #[error(transparent)] + Layout(#[from] gguf::write::WriteError), + #[error(transparent)] + Header(#[from] gguf::LoadError), + #[error(transparent)] + Tokenizer(#[from] tokenizer::Error), + #[error("{0}")] + Refused(String), +} + +impl From for Error { + fn from(why: String) -> Error { + Error::Refused(why) + } +} + +/// One tensor of the split: its declaration and how its bytes are made. +pub struct Planned { + pub decl: TensorDecl, + pub make: Make, +} + +/// How a planned tensor's bytes are made. +pub enum Make { + /// The mapped tensor, from its sources as the plan places them. + Tensor(Mapped), + /// The `.scale` beside the tensor `of`: its tensor scales, one a source for a stack. + Scale { of: String }, +} + +/// The folder as both readers take it: its config, its tensors, the quantization plan, the +/// planned tensors, the header's keys and serving's tokenizer. +pub struct Read { + pub checkpoint: Checkpoint, + pub folder: Folder, + pub plan: Plan, + pub planned: Vec, + pub kvs: Vec<(String, Value)>, + pub tokenizer: tokenizer::Tokenizer, +} + +/// The folder `dir` read and checked: `config.json`, its tensors placed by its quantization +/// config and planned ([`plan_tensors`], `q8` as there), the header's keys (the model's and the +/// tokenizer's), and serving's tokenizer built from them and checked against Hugging Face's +/// ([`served_as_hf`]). +pub fn read(dir: &Path, q8: bool) -> Result { + let config_path = dir.join("config.json"); + let config: serde_json::Value = std::fs::read(&config_path) + .map_err(|e| e.to_string()) + .and_then(|b| serde_json::from_slice(&b).map_err(|e| e.to_string())) + .map_err(|e| format!("{}: {e}", config_path.display()))?; + let checkpoint = Checkpoint::parse(&config)?; + let folder = Folder::open(dir)?; + let entries: Vec<(String, Dtype, Vec)> = folder + .names() + .map(|n| { + folder + .raw(n) + .map(|r| (n.to_string(), r.dtype, r.shape.to_vec())) + }) + .collect::>()?; + let (plan, planned) = plan_tensors(&checkpoint, &Config::read(dir)?, &entries, q8)?; + let mut kvs = checkpoint.header( + planned + .iter() + .map(|p| (p.decl.name.as_str(), p.decl.dims.as_slice())), + &folder, + )?; + let vocab = hf::vocab::keys(dir, &checkpoint.vocab)?; + kvs.extend(vocab.keys.iter().cloned()); + let tokenizer = served_as_hf(&kvs, &vocab)?; + Ok(Read { + checkpoint, + folder, + plan, + planned, + kvs, + tokenizer, + }) +} + +/// A folder as a load opens it ([`open`]). +pub struct Opened { + /// The header `prepare` writes as its split's first shard, with every tensor of the split + /// and none of their bytes, named by the folder's path. + pub header: Split, + /// Every tensor of the header but the PLE table and the `.scale` tensors, whose scales each + /// part carries. + pub source: part::Checkpoint, + /// The PLE table's rows where the folder stores them, when the model has the table. + pub ple: Option, + /// Serving's tokenizer, checked against Hugging Face's. + pub tokenizer: tokenizer::Tokenizer, +} + +/// The folder `dir` opened for a load, every part in its stored format (no Q8_0 rewrite): its +/// header ([`Split::header_only`] of [`read`]'s keys and planned tensors), its card tensors as a +/// part source, and its PLE table's rows ([`ple_rows`]). Any refusal of [`read`] or +/// [`ple_rows`] is the folder's. +pub fn open(dir: &Path) -> Result { + let Read { + checkpoint, + folder, + plan, + planned, + kvs, + tokenizer, + } = read(dir, false)?; + let layout = Layout::new(&kvs, planned.iter().map(|p| p.decl.clone()).collect())?; + let header = Split::header_only(dir.to_path_buf(), &layout)?; + let mut made = BTreeMap::new(); + let mut ple = None; + for Planned { decl, make } in planned { + let Make::Tensor(m) = make else { + continue; + }; + if m.role == Use::Rows { + ple = Some(ple_rows(&folder, &plan, &decl, &m)?); + continue; + } + let step = (!m.steps.is_empty()).then(|| { + let (g, name, steps) = (checkpoint.geometry.clone(), decl.name.clone(), m.steps); + Box::new(move |n: &mut modelopt::Normal| { + steps + .iter() + .try_for_each(|s| s.apply(&g, &name, n)) + .map_err(|e| e.to_string()) + }) as part::Step + }); + made.insert( + decl.name, + Made { + sources: m.sources, + role: m.role, + dims: m.dims, + step, + }, + ); + } + Ok(Opened { + header, + source: part::Checkpoint::new(folder, plan, made)?, + ple, + tokenizer, + }) +} + +/// The row table `decl`, mapped as `m`, where `folder` stores it: each part's byte range in its +/// shard. Only F8 rows are stored as the program reads them (`modelopt::fp8_rows_scale`), so a +/// table of another type, a part that is not `decl`'s rows wide, parts at different tensor +/// scales, and parts that do not add up to the table's rows are refused by name. +fn ple_rows( + folder: &Folder, + plan: &Plan, + decl: &TensorDecl, + m: &Mapped, +) -> Result { + let name = &decl.name; + let ty = GgmlType::from_u32(decl.type_id); + if ty != GgmlType::F8E4M3_ROWS { + return Err(format!( + "{name}: a folder load reads the PLE table's rows where the folder stores them, which \ + holds F8 rows as the program reads them; {} ({ty}) is stored in another layout: \ + prepare the folder into a split", + m.sources[0] + ) + .into()); + } + let (row_bytes, rows) = match decl.dims[..] { + [width, rows] => (width, rows), + _ => return Err(format!("{name}: dims {:?} are not a table's", decl.dims).into()), + }; + let get = |n: &str| folder.raw(n); + let mut scale: Option = None; + let mut runs = Vec::with_capacity(m.sources.len()); + let mut held = 0u64; + for (source, placed) in m.placed(plan)? { + let (s, _) = modelopt::fp8_rows_scale(&get, source, placed)?; + if let Some(t) = scale.filter(|&t| t != s) { + return Err(format!("{source}: tensor scale {s}, the table's {t}").into()); + } + scale = Some(s); + let shape = folder.raw(source)?.shape; + let &[part_rows, width] = shape else { + return Err(format!("{source}: shape {shape:?} is not a table part's").into()); + }; + if width as u64 != row_bytes { + return Err( + format!("{source}: rows of {width} values, the table's {row_bytes}").into(), + ); + } + held += part_rows as u64; + runs.push(folder.file_range(source)?); + } + if held != rows { + return Err(format!("{name}: its parts hold {held} rows, the table {rows}").into()); + } + Ok(StoredRows { + row_bytes, + runs, + scale: scale.ok_or_else(|| format!("{name}: a table of no parts"))?, + }) +} + +/// The folder's `entries` (name, dtype, shape) placed by its `config`, and each tensor the map +/// makes of them declared in the type its placed format and role give, its `.scale` beside it +/// where the type reads one. An unquantized matrix the map reads as Q8_0 stays in its stored +/// format unless `q8` asks for the rewrite; F32 matrices need that rewrite. Unquantized expert +/// stacks are unsupported. +pub fn plan_tensors( + checkpoint: &Checkpoint, + config: &Config, + entries: &[(String, Dtype, Vec)], + q8: bool, +) -> Result<(Plan, Vec), Error> { + let plan = quant_config::plan( + config, + entries + .iter() + .map(|(n, d, s)| (n.as_str(), *d, s.as_slice())), + crate::arch::module_class, + ); + let logical: Vec<(&str, Vec)> = entries + .iter() + .filter_map( + |(n, _, s)| match (plan.placed.get(n), plan.refused.contains_key(n)) { + (Some(p), _) => Some((n.as_str(), p.format.logical(s))), + (None, true) => Some((n.as_str(), s.clone())), + (None, false) => None, + }, + ) + .collect(); + let mapped = checkpoint.map(logical.iter().map(|(n, s)| (*n, s.as_slice())))?; + let mut planned = Vec::new(); + for (name, mut m) in mapped { + let source = &m.sources[0]; + let format = modelopt::format_of(&m.placed(&plan)?)?; + if m.role == Use::Matrix(Plain::Q8_0) && matches!(format, Format::Bf16 | Format::F32) { + if m.dims.len() == 3 { + return Err(format!( + "{source}: {name}: {format} expert stacks are unsupported by the converter" + ) + .into()); + } + if !q8 { + if format == Format::F32 { + return Err(format!( + "{source}: {name} is stored {format}, a form its card site does not read; {PREPARE_Q8}=1 writes Q8_0" + ) + .into()); + } + m.role = Use::Matrix(Plain::Bf16); + } + } + let type_id = modelopt::type_of(format, m.role).ok_or_else(|| { + format!( + "{source}: {format} read as {:?} has no GGUF type here", + m.role + ) + })?; + let nbytes = size_of(type_id, &m.dims).ok_or_else(|| { + format!( + "{name}: dims {:?} are not whole blocks of type {type_id}", + m.dims + ) + })?; + // One tensor scale an expert of a stack; one for a row table, whatever its parts. + let scales = GgmlType::from_u32(type_id) + .reads_scale() + .then_some(match m.role { + Use::Rows => 1, + Use::Matrix(_) => m.sources.len(), + }); + let decl = TensorDecl { + name: name.clone(), + dims: m.dims.clone(), + type_id, + nbytes, + }; + planned.push(Planned { + decl, + make: Make::Tensor(m), + }); + if let Some(n) = scales { + let n = u64::try_from(n).map_err(|_| format!("{name}: scale count exceeds u64"))?; + let nbytes = size_of(0, &[n]) + .ok_or_else(|| format!("{name}: scale byte count overflows u64"))?; + let stem = name.strip_suffix(".weight").unwrap_or(&name); + let decl = TensorDecl { + name: format!("{stem}.scale"), + dims: vec![n], + type_id: 0, + nbytes, + }; + planned.push(Planned { + decl, + make: Make::Scale { of: name }, + }); + } + } + Ok((plan, planned)) +} + +/// Serving's tokenizer, built from the header's keys `kvs`, which must preserve HF's split, +/// rendering, special ids, automatic BOS/EOS and generation stop sets. +pub fn served_as_hf( + kvs: &[(String, Value)], + vocab: &hf::vocab::Keys, +) -> Result { + let served = tokenizer::Tokenizer::from_keys(kvs)?; + let mut split = served.special_tokens().to_vec(); + split.sort_unstable(); + if split != vocab.added { + return Err(format!( + "serving splits text on tokens {split:?}, HF on its added tokens {:?}", + vocab.added + ) + .into()); + } + if let Some(id) = vocab.stop.iter().find(|id| !served.eog().contains(id)) { + return Err(format!( + "HF's generate stops at token {id}, where serving does not end generation" + ) + .into()); + } + if let Some(id) = served.eog().iter().find(|id| !vocab.stop.contains(id)) { + return Err(format!("serving stops at token {id}, where HF's generate continues").into()); + } + let expected = |key: &str| vocab.keys.iter().find(|(k, _)| k == key).map(|(_, v)| v); + let Some(Value::Array(types)) = expected("tokenizer.ggml.token_type") else { + return Err("HF tokenizer has no tokenizer.ggml.token_type" + .to_string() + .into()); + }; + if types.len() != served.n_vocab() { + return Err("serving vocabulary size differs from HF's" + .to_string() + .into()); + } + for (id, ty) in types.iter().enumerate() { + let id = u32::try_from(id).map_err(|_| format!("token {id} past u32"))?; + let control = served + .attr(id) + .is_some_and(|a| a & (tokenizer::attr::CONTROL | tokenizer::attr::UNKNOWN) != 0); + if control != (*ty == Value::I32(3)) { + return Err( + format!("serving control/rendering at token {id} differs from HF's").into(), + ); + } + } + for (key, actual) in [ + ("tokenizer.ggml.add_bos_token", served.add_bos()), + ("tokenizer.ggml.add_eos_token", served.add_eos()), + ] { + if actual != expected(key).and_then(Value::as_bool).unwrap_or(false) { + return Err(format!("serving {key} differs from HF's").into()); + } + } + let specials = served.specials(); + for (key, actual) in [ + ("tokenizer.ggml.bos_token_id", specials.bos), + ("tokenizer.ggml.eos_token_id", specials.eos), + ("tokenizer.ggml.padding_token_id", specials.pad), + ("tokenizer.ggml.unknown_token_id", specials.unk), + ("tokenizer.ggml.seperator_token_id", specials.sep), + ] { + if actual.is_none() + && matches!( + key, + "tokenizer.ggml.bos_token_id" | "tokenizer.ggml.eos_token_id" + ) + { + return Err(format!("serving requires {key}").into()); + } + if actual.map(u64::from) != expected(key).and_then(Value::as_u64) { + return Err(format!("serving {key} differs from HF's").into()); + } + } + Ok(served) +} + +/// The bytes of a tensor of `type_id` and `dims`, its blocks as the type's row sizes them. +pub fn size_of(type_id: u32, dims: &[u64]) -> Option { + let ty = GgmlType::from_u32(type_id); + let (blck, size) = ty.blck_size().zip(ty.type_size())?; + if dims.is_empty() || !dims[0].is_multiple_of(blck) { + return None; + } + gguf::tensor_bytes(blck, size, dims) +} + +#[cfg(test)] +mod tests { + use hf::quant_config::plan; + + use super::*; + + /// A table in F8 parts across two shards is read where the folder stores it: each part's + /// byte range in its shard, in the parts' order, at the one tensor scale. Parts at unequal + /// scales, a part of another width, parts short of the table's rows and a table stored in + /// another layout (FP4-Local's NVFP4 rows) are refused by name. + #[test] + fn a_table_s_rows_are_read_where_the_folder_stores_them() { + let dir = std::env::temp_dir().join(format!("folder-ple-{}", std::process::id())); + let hq = serde_json::json!({"quantization": {"quant_algo": "MIXED_PRECISION", + "quantized_layers": {"e": {"quant_algo": "FP8"}}}}); + let make = |second_scale: f32, width: usize| { + std::fs::create_dir_all(&dir).unwrap(); + let (one, two) = (1.5f32.to_le_bytes(), second_scale.to_le_bytes()); + let (a, b) = (vec![0x38u8; 2 * 160], vec![0x40u8; 3 * width]); + let first: [(&str, &str, &[usize], &[u8]); 2] = [ + ("e.shard_0.weight", "F8_E4M3", &[2, 160], &a), + ("e.shard_0.weight_scale", "F32", &[1], &one), + ]; + let second: [(&str, &str, &[usize], &[u8]); 2] = [ + ("e.shard_1.weight", "F8_E4M3", &[3, width], &b), + ("e.shard_1.weight_scale", "F32", &[1], &two), + ]; + std::fs::write(dir.join("a.safetensors"), safetensors::write(&first)).unwrap(); + std::fs::write(dir.join("b.safetensors"), safetensors::write(&second)).unwrap(); + std::fs::write( + dir.join(Folder::INDEX), + r#"{"weight_map": {"e.shard_0.weight": "a.safetensors", + "e.shard_0.weight_scale": "a.safetensors", + "e.shard_1.weight": "b.safetensors", + "e.shard_1.weight_scale": "b.safetensors"}}"#, + ) + .unwrap(); + let folder = Folder::open(&dir).unwrap(); + let entries: Vec<(String, Dtype, Vec)> = folder + .names() + .map(|n| { + let r = folder.raw(n).unwrap(); + (n.to_string(), r.dtype, r.shape.to_vec()) + }) + .collect(); + let p = plan( + &Config::parse(&serde_json::json!({}), Some(&hq)).unwrap(), + entries + .iter() + .map(|(n, d, s)| (n.as_str(), *d, s.as_slice())), + crate::arch::module_class, + ); + (folder, p) + }; + let m = Mapped { + sources: vec!["e.shard_0.weight".into(), "e.shard_1.weight".into()], + role: Use::Rows, + steps: vec![], + dims: vec![160, 5], + }; + let decl = |ty: GgmlType, rows: u64| TensorDecl { + name: "per_layer_token_embd.weight".into(), + dims: vec![160, rows], + type_id: ty.as_u32(), + nbytes: 160 * rows, + }; + let (folder, p) = make(1.5, 160); + let got = ple_rows(&folder, &p, &decl(GgmlType::F8E4M3_ROWS, 5), &m).unwrap(); + assert_eq!((got.row_bytes, got.scale), (160, 1.5)); + let read = |(path, range): &(std::path::PathBuf, std::ops::Range)| { + std::fs::read(path).unwrap()[range.start as usize..range.end as usize].to_vec() + }; + assert_eq!(read(&got.runs[0]), vec![0x38u8; 320]); + assert_eq!(read(&got.runs[1]), vec![0x40u8; 480]); + let refusal = |folder: &Folder, p: &Plan, ty, rows| { + ple_rows(folder, p, &decl(ty, rows), &m) + .err() + .expect("refused") + .to_string() + }; + let e = refusal(&folder, &p, GgmlType::F8E4M3_ROWS, 6); + assert!(e.contains("its parts hold 5 rows, the table 6"), "{e}"); + let e = refusal(&folder, &p, GgmlType::NVFP4_ROWS, 5); + assert!(e.contains("prepare the folder into a split"), "{e}"); + let (folder, p) = make(2.0, 160); + let e = refusal(&folder, &p, GgmlType::F8E4M3_ROWS, 5); + assert!(e.contains("e.shard_1.weight: tensor scale 2"), "{e}"); + let (folder, p) = make(1.5, 96); + let e = refusal(&folder, &p, GgmlType::F8E4M3_ROWS, 5); + assert!(e.contains("rows of 96 values, the table's 160"), "{e}"); + std::fs::remove_dir_all(&dir).unwrap(); + } +} diff --git a/crates/model-hf/src/bin/prepare.rs b/crates/model-hf/src/bin/prepare.rs index 2aa069cc8..da902da2f 100644 --- a/crates/model-hf/src/bin/prepare.rs +++ b/crates/model-hf/src/bin/prepare.rs @@ -31,12 +31,10 @@ use std::process::ExitCode; use std::time::Instant; use bloomery_levers::PREPARE_Q8; -use gguf::write::{Layout, TensorDecl, Writer}; -use gguf::{GgmlType, Value}; -use hf::modelopt::{self, Normal, Plain, Use}; -use hf::quant_config::{self, Config, Format, Plan}; -use model_hf::arch::qwen35moe::{Checkpoint, Mapped}; -use safetensors::{Dtype, Folder}; +use gguf::Value; +use gguf::write::{Layout, Writer}; +use hf::modelopt::{self, Normal, Use}; +use model_hf::arch::qwen35moe::folder::{self, Make, Planned}; const USAGE: &str = "usage: prepare [--shard-gib ] [--keep-bf16]"; @@ -58,19 +56,6 @@ fn main() -> ExitCode { } } -/// One tensor of the split: its declaration and how its bytes are made. -struct Planned { - decl: TensorDecl, - make: Make, -} - -enum Make { - /// The mapped tensor, from its sources as the plan places them. - Tensor(Mapped), - /// The `.scale` beside the tensor `of`: its tensor scales, one a source for a stack. - Scale { of: String }, -} - /// The split of the folder `args` name; `q8` (`BLOOMERY_PREPARE_Q8`) writes its BF16 matrices the /// map reads as Q8_0, and its F32 matrices, in Q8_0. fn run(args: &[String], q8: bool) -> Res<()> { @@ -98,33 +83,14 @@ fn run(args: &[String], q8: bool) -> Res<()> { format!("--shard-gib {shard_gib}: not a shard size of 1 GiB or more in u64 bytes") })?; let dir = Path::new(dir); - let config: serde_json::Value = - serde_json::from_slice(&std::fs::read(dir.join("config.json"))?)?; - let checkpoint = Checkpoint::parse(&config)?; - let folder = Folder::open(dir)?; - let entries: Vec<(String, Dtype, Vec)> = folder - .names() - .map(|n| { - folder - .raw(n) - .map(|r| (n.to_string(), r.dtype, r.shape.to_vec())) - }) - .collect::>()?; - let (plan, planned) = plan_tensors(&checkpoint, &Config::read(dir)?, &entries, q8)?; - - let mut kvs = checkpoint.header( - planned - .iter() - .map(|p| (p.decl.name.as_str(), p.decl.dims.as_slice())), - &folder, - )?; - let vocab = hf::vocab::keys(dir, &checkpoint.vocab)?; - kvs.extend(vocab.keys.iter().cloned()); - served_as_hf( - &kvs, - &vocab, - Path::new(&format!("{prefix}.tokenizer.gguf.part")), - )?; + let folder::Read { + checkpoint, + folder, + plan, + planned, + kvs, + .. + } = folder::read(dir, q8)?; let tensors = planned.len(); let shards = plan_shards(planned, &kvs, limit)?; @@ -235,105 +201,6 @@ fn run(args: &[String], q8: bool) -> Res<()> { Ok(()) } -/// The folder's `entries` (name, dtype, shape) placed by its `config`, and each tensor the map -/// makes of them declared in the type its placed format and role give, its `.scale` beside it -/// where the type reads one. An unquantized matrix the map reads as Q8_0 stays in its stored -/// format unless `q8` asks for the rewrite; F32 matrices need that rewrite. Unquantized expert -/// stacks are unsupported. -fn plan_tensors( - checkpoint: &Checkpoint, - config: &Config, - entries: &[(String, Dtype, Vec)], - q8: bool, -) -> Res<(Plan, Vec)> { - let plan = quant_config::plan( - config, - entries - .iter() - .map(|(n, d, s)| (n.as_str(), *d, s.as_slice())), - model_hf::arch::module_class, - ); - let logical: Vec<(&str, Vec)> = entries - .iter() - .filter_map( - |(n, _, s)| match (plan.placed.get(n), plan.refused.contains_key(n)) { - (Some(p), _) => Some((n.as_str(), p.format.logical(s))), - (None, true) => Some((n.as_str(), s.clone())), - (None, false) => None, - }, - ) - .collect(); - let mapped = checkpoint.map(logical.iter().map(|(n, s)| (*n, s.as_slice())))?; - let mut planned = Vec::new(); - for (name, mut m) in mapped { - let source = &m.sources[0]; - let format = modelopt::format_of(&m.placed(&plan)?)?; - if m.role == Use::Matrix(Plain::Q8_0) && matches!(format, Format::Bf16 | Format::F32) { - if m.dims.len() == 3 { - return Err(format!( - "{source}: {name}: {format} expert stacks are unsupported by the converter" - ) - .into()); - } - if !q8 { - if format == Format::F32 { - return Err(format!( - "{source}: {name} is stored {format}, a form its card site does not read; {PREPARE_Q8}=1 writes Q8_0" - ) - .into()); - } - m.role = Use::Matrix(Plain::Bf16); - } - } - let type_id = modelopt::type_of(format, m.role).ok_or_else(|| { - format!( - "{source}: {format} read as {:?} has no GGUF type here", - m.role - ) - })?; - let nbytes = size_of(type_id, &m.dims).ok_or_else(|| { - format!( - "{name}: dims {:?} are not whole blocks of type {type_id}", - m.dims - ) - })?; - // One tensor scale an expert of a stack; one for a row table, whatever its parts. - let scales = GgmlType::from_u32(type_id) - .reads_scale() - .then_some(match m.role { - Use::Rows => 1, - Use::Matrix(_) => m.sources.len(), - }); - let decl = TensorDecl { - name: name.clone(), - dims: m.dims.clone(), - type_id, - nbytes, - }; - planned.push(Planned { - decl, - make: Make::Tensor(m), - }); - if let Some(n) = scales { - let n = u64::try_from(n).map_err(|_| format!("{name}: scale count exceeds u64"))?; - let nbytes = size_of(0, &[n]) - .ok_or_else(|| format!("{name}: scale byte count overflows u64"))?; - let stem = name.strip_suffix(".weight").unwrap_or(&name); - let decl = TensorDecl { - name: format!("{stem}.scale"), - dims: vec![n], - type_id: 0, - nbytes, - }; - planned.push(Planned { - decl, - make: Make::Scale { of: name }, - }); - } - } - Ok((plan, planned)) -} - fn plan_shards( planned: Vec, kvs: &[(String, Value)], @@ -384,103 +251,14 @@ fn plan_shards( .collect() } -/// Serving's tokenizer, which must preserve HF's split, rendering, special ids, automatic BOS/EOS -/// and generation stop sets. The header-only `probe` is removed before any shard is written. -fn served_as_hf( - kvs: &[(String, Value)], - vocab: &hf::vocab::Keys, - probe: &Path, -) -> Res { - let open = || -> Res { - let layout = Layout::new(kvs, Vec::new())?; - Writer::new(File::create(probe)?, layout)?.finish()?; - Ok(tokenizer::Tokenizer::from_gguf(probe)?) - }; - let served = open(); - if probe.exists() { - std::fs::remove_file(probe)?; - } - let served = served?; - let mut split = served.special_tokens().to_vec(); - split.sort_unstable(); - if split != vocab.added { - return Err(format!( - "serving splits text on tokens {split:?}, HF on its added tokens {:?}", - vocab.added - ) - .into()); - } - if let Some(id) = vocab.stop.iter().find(|id| !served.eog().contains(id)) { - return Err(format!( - "HF's generate stops at token {id}, where serving does not end generation" - ) - .into()); - } - if let Some(id) = served.eog().iter().find(|id| !vocab.stop.contains(id)) { - return Err(format!("serving stops at token {id}, where HF's generate continues").into()); - } - let expected = |key: &str| vocab.keys.iter().find(|(k, _)| k == key).map(|(_, v)| v); - let Some(Value::Array(types)) = expected("tokenizer.ggml.token_type") else { - return Err("HF tokenizer has no tokenizer.ggml.token_type".into()); - }; - if types.len() != served.n_vocab() { - return Err("serving vocabulary size differs from HF's".into()); - } - for (id, ty) in types.iter().enumerate() { - let id = u32::try_from(id)?; - let control = served - .attr(id) - .is_some_and(|a| a & (tokenizer::attr::CONTROL | tokenizer::attr::UNKNOWN) != 0); - if control != (*ty == Value::I32(3)) { - return Err( - format!("serving control/rendering at token {id} differs from HF's").into(), - ); - } - } - for (key, actual) in [ - ("tokenizer.ggml.add_bos_token", served.add_bos()), - ("tokenizer.ggml.add_eos_token", served.add_eos()), - ] { - if actual != expected(key).and_then(Value::as_bool).unwrap_or(false) { - return Err(format!("serving {key} differs from HF's").into()); - } - } - let specials = served.specials(); - for (key, actual) in [ - ("tokenizer.ggml.bos_token_id", specials.bos), - ("tokenizer.ggml.eos_token_id", specials.eos), - ("tokenizer.ggml.padding_token_id", specials.pad), - ("tokenizer.ggml.unknown_token_id", specials.unk), - ("tokenizer.ggml.seperator_token_id", specials.sep), - ] { - if actual.is_none() - && matches!( - key, - "tokenizer.ggml.bos_token_id" | "tokenizer.ggml.eos_token_id" - ) - { - return Err(format!("serving requires {key}").into()); - } - if actual.map(u64::from) != expected(key).and_then(Value::as_u64) { - return Err(format!("serving {key} differs from HF's").into()); - } - } - Ok(served) -} - -/// The bytes of a tensor of `type_id` and `dims`, its blocks as the type's row sizes them. -fn size_of(type_id: u32, dims: &[u64]) -> Option { - let ty = GgmlType::from_u32(type_id); - let (blck, size) = ty.blck_size().zip(ty.type_size())?; - if dims.is_empty() || !dims[0].is_multiple_of(blck) { - return None; - } - gguf::tensor_bytes(blck, size, dims) -} - #[cfg(test)] mod tests { - use gguf::GENERAL_ARCHITECTURE; + use gguf::write::TensorDecl; + use gguf::{GENERAL_ARCHITECTURE, GgmlType}; + use hf::quant_config::{Config, Format}; + use model_hf::arch::qwen35moe::Checkpoint; + use model_hf::arch::qwen35moe::folder::{plan_tensors, served_as_hf}; + use safetensors::Dtype; #[test] fn shard_budget_counts_scales_metadata_and_padding() { use super::*; @@ -564,8 +342,6 @@ mod tests { fn the_served_tokenizer_splits_and_stops_as_hf() { use super::*; - let probe = - std::env::temp_dir().join(format!("prepare-served-{}.gguf", std::process::id())); let stop_key = "tokenizer.bloomery.eog_token_ids"; for (second, added, stop, explicit, refused) in [ ("b", vec![2], vec![2], false, None), @@ -649,12 +425,11 @@ mod tests { added, stop, }; - match (served_as_hf(&keys, &vocab, &probe).map(|_| ()), refused) { + match (served_as_hf(&keys, &vocab).map(|_| ()), refused) { (Ok(()), None) => {} (Err(e), Some(what)) if e.to_string().contains(what) => {} (other, _) => panic!("{second} {:?}: {other:?}", vocab.stop), } - assert!(!probe.exists()); if explicit && refused.is_none() { for (key, value, error) in [ ( @@ -698,9 +473,8 @@ mod tests { let mut changed = keys.clone(); changed.retain(|(k, _)| k != key); changed.push((key.into(), value)); - let err = served_as_hf(&changed, &vocab, &probe).err().unwrap(); + let err = served_as_hf(&changed, &vocab).err().unwrap(); assert!(err.to_string().contains(error), "{second} {key}: {err}"); - assert!(!probe.exists()); } for key in ["tokenizer.ggml.bos_token_id", "tokenizer.ggml.eos_token_id"] { let mut absent = hf::vocab::Keys { @@ -709,12 +483,11 @@ mod tests { stop: vocab.stop.clone(), }; absent.keys.retain(|(k, _)| k != key); - let err = served_as_hf(&absent.keys, &absent, &probe).err().unwrap(); + let err = served_as_hf(&absent.keys, &absent).err().unwrap(); assert!( err.to_string().contains(&format!("serving requires {key}")), "{err}" ); - assert!(!probe.exists()); } let mut both = keys.clone(); let mut expected = hf::vocab::Keys { @@ -724,14 +497,14 @@ mod tests { }; both.push(("tokenizer.ggml.add_bos_token".into(), Value::Bool(true))); assert!( - served_as_hf(&both, &expected, &probe) + served_as_hf(&both, &expected) .err() .unwrap() .to_string() .contains("add_bos_token") ); expected.keys = both.clone(); - assert!(served_as_hf(&both, &expected, &probe).is_ok()); + assert!(served_as_hf(&both, &expected).is_ok()); } } } @@ -766,9 +539,7 @@ mod tests { let config = serde_json::from_slice(&std::fs::read(dir.join("config.json")).unwrap()); let checkpoint = Checkpoint::parse(&config.unwrap()).unwrap(); let vocab = hf::vocab::keys(&dir, &checkpoint.vocab).unwrap(); - let probe = - std::env::temp_dir().join(format!("prepare-corpus-{}.gguf", std::process::id())); - let served = served_as_hf(&vocab.keys, &vocab, &probe).unwrap(); + let served = served_as_hf(&vocab.keys, &vocab).unwrap(); let cases = corpus["cases"].as_array().unwrap(); assert!(cases.len() >= 2000, "{} cases in {path}", cases.len()); let (mut ids, mut failures) = (0, Vec::new()); diff --git a/crates/model-hf/src/part.rs b/crates/model-hf/src/part.rs index f376ff919..a0eda9a2d 100644 --- a/crates/model-hf/src/part.rs +++ b/crates/model-hf/src/part.rs @@ -27,6 +27,11 @@ pub enum Error { /// A model's move refused the tensor; the reason names it. #[error("{0}")] Step(String), + /// A stored tensor's file pages could not be released. + #[error(transparent)] + Release(#[from] model::placement::PlacementError), + #[error(transparent)] + Tensors(#[from] safetensors::Error), } impl From for Error { @@ -205,6 +210,27 @@ impl Source for Checkpoint { ty.reads_scale().then_some(n.scale), )) } + + /// Every stored tensor the part was read from, its values and its companions (scales, the + /// packed shape), each the whole pages inside its bytes: a read normalizes into bytes of its + /// own ([`Part::made`]), so no later reader of the part takes them from the file. + fn release(&self, p: &Planned) -> Result<(), Error> { + let name = p.info.name.as_str(); + let m = self + .made + .get(name) + .ok_or_else(|| refuse(name, "is not one the map makes"))?; + for (source, placed) in self.placed(m) { + for stored in + std::iter::once(source).chain(placed.companions.iter().map(String::as_str)) + { + let (path, map, at) = self.folder.mapped(stored)?; + // SAFETY: `Folder::mapped` gives the shard file's own read-only mapping whole. + unsafe { model::placement::host_lock::release_file_pages(&path, map, at) }?; + } + } + Ok(()) + } } /// The whole tensor `name` of `ty` and `dims` as a header entry; dims that are not whole blocks @@ -571,7 +597,7 @@ pub(crate) mod tests { /// whole, uncut; at two and four ranks along every dim of each format, a stack's experts and /// a tensor the map moves. A part off the card type's blocks is refused by both sources; /// one that splits FP8's 128-row scale blocks by the folder alone, naming its scale, in its - /// plan. + /// plan. A part read again after its file pages' release gives the same bytes. #[test] fn a_folder_and_its_gguf_give_every_rank_the_same_part() { let dir = temp_dir("same"); @@ -669,6 +695,14 @@ pub(crate) mod tests { "{e}" ); println!("SAME {same} rank parts"); + for (name, _) in cases { + let before = read_one(f, rule(name, 0, 0, 1)).unwrap(); + for p in f.plan(rule(name, 0, 0, 1)).unwrap() { + f.release(&p).unwrap(); + } + let after = read_one(f, rule(name, 0, 0, 1)).unwrap(); + assert_eq!(after, before, "{name} read again after its pages' release"); + } std::fs::remove_dir_all(&dir).unwrap(); } diff --git a/crates/model/src/part.rs b/crates/model/src/part.rs index 7f191c750..83fb98ac8 100644 --- a/crates/model/src/part.rs +++ b/crates/model/src/part.rs @@ -10,6 +10,7 @@ //! cut of a stack's experts. use std::ops::Range; +use std::path::PathBuf; use gguf::{GgmlType, Split, TensorInfo}; @@ -103,15 +104,17 @@ impl Part<'_> { Bytes::Made(bytes) => vec![bytes], } } +} - /// The bytes the part holds in memory of its own, beside the file's mapping. - #[must_use] - pub fn held(&self) -> usize { - match &self.bytes { - Bytes::File(_) => 0, - Bytes::Made(bytes) => bytes.len(), - } - } +/// A row-gathered table's rows where a source stores them, for a reader that maps them there +/// (a checkpoint folder's PLE table, its parts in the folder's shard files): rows of `row_bytes` +/// bytes as the program reads them, each run a byte range of whole rows in a file, in row +/// order, every row at the one tensor scale `scale`. +#[derive(Clone, Debug, PartialEq)] +pub struct StoredRows { + pub row_bytes: u64, + pub runs: Vec<(PathBuf, Range)>, + pub scale: f32, } /// Where a load reads its tensors. @@ -131,6 +134,13 @@ pub trait Source { /// The bytes of the part `p` planned ([`Source::plan`]) and the tensor scales its type reads; /// a tensor no longer in the source is refused by name. fn read(&self, p: &Planned) -> Result, Self::Error>; + + /// Drop the file pages the source read part `p` from, once a load has copied its bytes and no + /// later reader takes them from the file. A source whose loads release their own pages (a + /// split's placed load, `PageDrop`) drops none here. + fn release(&self, _p: &Planned) -> Result<(), Self::Error> { + Ok(()) + } } impl Source for Split { diff --git a/crates/model/src/placement/host_lock.rs b/crates/model/src/placement/host_lock.rs index bc398e40d..650398caf 100644 --- a/crates/model/src/placement/host_lock.rs +++ b/crates/model/src/placement/host_lock.rs @@ -787,12 +787,17 @@ impl<'a> FileMapping<'a> { FileMapping(map) } - /// Shard `s` of `split`'s mapping. + /// Shard `s` of `split`'s mapping; a shard that is no file's mapping (a + /// header alone, `Split::header_only`) is refused by name. pub(crate) fn shard(split: &'a Split, s: usize) -> Result, PlacementError> { let g = shard_of(split, s)?; - // SAFETY: `Split::open` opens every shard with `Gguf::open`, a - // read-only file mapping (`Weights::Mapped`), and a split has no - // other constructor. + if !g.file_backed() { + return Err(PlacementError::Host(format!( + "shard {s} is a header alone, no file's mapping: it holds no page of the file" + ))); + } + // SAFETY: a file-backed shard's mapping is the file's own read-only + // mapping (`Gguf::file_backed`, `Split::open`'s `Gguf::open`). Ok(unsafe { FileMapping::new(g.mapping()) }) } @@ -808,9 +813,10 @@ impl<'a> FileMapping<'a> { /// `map` is the file's own mapping ([`FileMapping`]); `None` is a file this /// process maps no page of there (a resident sidecar), and only the page /// cache's copy goes. The one owner of page release: [`PageDrop`] and -/// `r8file::verify` call it. +/// `r8file::verify` call it, and [`release_file_pages`] for a file outside a +/// split. pub(crate) fn drop_pages( - name: &HostFile, + name: &dyn fmt::Display, file: &File, map: Option>, at: Range, @@ -863,6 +869,28 @@ pub(crate) fn drop_pages( Ok(end - first) } +/// [`PageDrop::release`] for a file outside a split (a checkpoint folder's +/// shard): the whole pages inside bytes `at` of the file at `path`, whose +/// whole mapping is `map`, dropped from this process's page tables and then +/// from the page cache; the bytes dropped. For bytes a load has copied that +/// no later reader takes from the file. +/// +/// # Safety +/// +/// `map` is the whole of the file's own read-only `MAP_SHARED` mapping, file +/// offset `o` at index `o` ([`FileMapping::new`]'s contract). +pub unsafe fn release_file_pages( + path: &std::path::Path, + map: &[u8], + at: Range, +) -> Result { + let file = File::open(path) + .map_err(|e| PlacementError::Host(format!("open {}: {e}", path.display())))?; + // SAFETY: the caller's contract is `FileMapping::new`'s. + let map = unsafe { FileMapping::new(map) }; + drop_pages(&path.display(), &file, Some(map), at) +} + /// The line a page drop prints once when the platform has no /// `posix_fadvise(DONTNEED)`: the page cache keeps the pages the call would /// have dropped. @@ -1403,6 +1431,25 @@ mod tests { println!("walk: a chunk's panic is its named refusal, its taken span handed back ({e})"); } + /// A header alone (`Split::header_only`) is no file's mapping: a page release over it is + /// refused by name before any advice, as its bytes are a heap copy `MADV_DONTNEED` would zero. + #[test] + fn a_header_alone_releases_no_page() { + let decl = TensorDecl { + name: "w".to_string(), + dims: vec![4], + type_id: 0, + nbytes: 16, + }; + let layout = Layout::new(&[], vec![decl]).unwrap(); + let split = Split::header_only(std::env::temp_dir().join("header-alone"), &layout).unwrap(); + let e = super::PageDrop::new(&split) + .release(0, 0..16) + .unwrap_err() + .to_string(); + assert!(e.contains("a header alone"), "{e}"); + } + /// A cut keeps every page once and in order, and its pieces differ by at /// most a page, into any count from one to the pages: over one run, runs /// shorter and longer than a piece (so cuts fall inside runs and on diff --git a/crates/safetensors/src/lib.rs b/crates/safetensors/src/lib.rs index 012a60d48..f0e159017 100644 --- a/crates/safetensors/src/lib.rs +++ b/crates/safetensors/src/lib.rs @@ -245,6 +245,24 @@ impl Safetensors { self.entries.get(name) } + /// Where `name`'s bytes lie in the file: the byte range from the file's start. + pub fn file_range(&self, name: &str) -> Result, Error> { + let e = self + .entries + .get(name) + .ok_or_else(|| Error::Missing(name.to_string()))?; + Ok((self.data + e.start) as u64..(self.data + e.end) as u64) + } + + /// The whole of the file's own read-only mapping, file offset `o` at index `o`; `None` for + /// bytes held on the heap ([`Safetensors::parse`]). + pub fn file_mapping(&self) -> Option<&[u8]> { + match &self.bytes { + Backing::Map(m) => Some(m), + Backing::Heap(_) => None, + } + } + /// `name` as stored: dtype, shape and the bytes in the file. pub fn raw(&self, name: &str) -> Result, Error> { let e = self @@ -373,6 +391,7 @@ pub fn f32_values(name: &str, raw: &Raw<'_>) -> Result, Error> { /// and folder names with no `\` or `:`, as `tensors/native-QB.safetensors`), and either mismatch, /// are refused by name. pub struct Folder { + dir: std::path::PathBuf, /// Each shard's file name and file, in the order the index first names them. shards: Vec<(String, Safetensors)>, /// A tensor's name to its shard. @@ -444,7 +463,34 @@ impl Folder { ))); } } - Ok(Folder { shards, by_name }) + Ok(Folder { + dir: dir.to_path_buf(), + shards, + by_name, + }) + } + + /// Where `name`'s bytes lie: its shard's path and the byte range in that file + /// ([`Safetensors::file_range`]), for a reader that maps or reads the file itself. + pub fn file_range( + &self, + name: &str, + ) -> Result<(std::path::PathBuf, std::ops::Range), Error> { + let (file, st) = self.shard(name)?; + Ok((self.dir.join(file), st.file_range(name)?)) + } + + /// Tensor `name`'s shard file, that file's whole read-only mapping + /// ([`Safetensors::file_mapping`]) and the tensor's byte range in it. + pub fn mapped( + &self, + name: &str, + ) -> Result<(std::path::PathBuf, &[u8], std::ops::Range), Error> { + let (file, st) = self.shard(name)?; + let map = st + .file_mapping() + .ok_or_else(|| bad(format!("{file} is no file's mapping")))?; + Ok((self.dir.join(file), map, st.file_range(name)?)) } /// Every tensor name, sorted. @@ -454,11 +500,16 @@ impl Folder { /// `name` as stored, from its shard. pub fn raw(&self, name: &str) -> Result, Error> { + self.shard(name)?.1.raw(name) + } + + /// The file name and file of the shard that holds `name`. + fn shard(&self, name: &str) -> Result<&(String, Safetensors), Error> { let &at = self .by_name .get(name) .ok_or_else(|| Error::Missing(name.to_string()))?; - self.shards[at].1.raw(name) + Ok(&self.shards[at]) } } diff --git a/crates/tokenizer/src/lib.rs b/crates/tokenizer/src/lib.rs index 5e83b562c..36812ee40 100644 --- a/crates/tokenizer/src/lib.rs +++ b/crates/tokenizer/src/lib.rs @@ -124,7 +124,16 @@ impl Tokenizer { /// Load the vocabulary from a GGUF file's header; for a split set, the /// first shard (the one that carries `tokenizer.ggml.*`). pub fn from_gguf(path: impl AsRef) -> Result { - let (vocab, encoding, hf_texts) = Vocab::load(path.as_ref())?; + let path = path.as_ref(); + let inv = + gguf::inventory_of(path).map_err(|e| Error::Gguf(path.display().to_string(), e))?; + Tokenizer::from_keys(&inv.meta) + } + + /// Load the vocabulary from a model header's metadata pairs (`gguf::Split::keys`): a + /// checkpoint folder's header, which no file holds, reads as a GGUF file's. + pub fn from_keys(keys: &[(String, gguf::Value)]) -> Result { + let (vocab, encoding, hf_texts) = Vocab::load(keys)?; let encoder = encoder::Encoder::new(encoding, &vocab)?; let spelling = encoder.spelling(); let specials = vocab.special_order(); diff --git a/crates/tokenizer/src/vocab.rs b/crates/tokenizer/src/vocab.rs index 8b9d3391b..12766c58d 100644 --- a/crates/tokenizer/src/vocab.rs +++ b/crates/tokenizer/src/vocab.rs @@ -3,7 +3,6 @@ //! applies by token text, and the special-token list the partition walks. use std::collections::{HashMap, HashSet}; -use std::path::Path; use gguf::Value; @@ -184,13 +183,13 @@ pub(crate) struct Vocab { } struct Meta<'a> { - inv: &'a gguf::Inventory, + keys: &'a [(String, Value)], read: Vec<(String, String)>, } impl<'a> Meta<'a> { fn get(&mut self, key: &str) -> Option<&'a Value> { - let v = self.inv.value(key); + let v = self.keys.iter().find(|(k, _)| k == key).map(|(_, v)| v); let shown = match v { None => "(absent)".to_string(), Some(Value::Array(a)) => format!("array of {}", a.len()), @@ -289,11 +288,11 @@ impl<'a> Meta<'a> { } impl Vocab { - pub(crate) fn load(path: &Path) -> Result<(Vocab, Encoding, Option>), Error> { - let inv = - gguf::inventory_of(path).map_err(|e| Error::Gguf(path.display().to_string(), e))?; + pub(crate) fn load( + keys: &[(String, Value)], + ) -> Result<(Vocab, Encoding, Option>), Error> { let mut m = Meta { - inv: &inv, + keys, read: Vec::new(), }; diff --git a/docs/BUILD.md b/docs/BUILD.md index bafec51be..728b0ab81 100644 --- a/docs/BUILD.md +++ b/docs/BUILD.md @@ -69,7 +69,7 @@ This builds the V4.1 CLI, the GPU kernels and the host expert tier into one bina |---|---|---| | `generate_ds41`, `bloomery-chat` | `cargo oxide build --arch sm_86 -- -p bloomery-gpu-gates --features deepseek41 --release --bin ` | DeepSeek-V4.1-Flash | | `bloomery-serve-ds41`, `bloomery-serve-qwen38` | the same, `--features deepseek41,gemma-embedding2` (`gemma-embedding2` attaches the optional embedding side seat) | DeepSeek-V4.1-Flash, Qwen3.8-Flash-Next | -| `bloomery-serve` | `… --features glm5next,clef,vision,mimo2,solar2,gemma-embedding2,hf --release --bin bloomery-serve` (`--model ds41\|qwen38\|glm\|qwen3\|mimo2\|solar2\|nemotron\|decide`; the ds41 and qwen38 seats are the two binaries above; `vision` adds V4.1's `--mmproj`) | DeepSeek-V4.1-Flash, Qwen3.8-Flash-Next, GLM-5.3-Flash, Qwen3.6-35B-A3B, Qwen3-30B-A3B, MiMo-V2.6-Flash, Solar-Open2-250B, Nemotron 3 Nano 4B, Clef-Flash, lev | +| `bloomery-serve` | `… --features glm5next,clef,vision,mimo2,solar2,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve` (`--model ds41\|qwen38\|glm\|qwen3\|mimo2\|solar2\|nemotron\|decide`; the ds41 and qwen38 seats are the two binaries above; `vision` adds V4.1's `--mmproj`; `checkpoint` opens a Qwen3.8 safetensors checkpoint folder as stored) | DeepSeek-V4.1-Flash, Qwen3.8-Flash-Next, GLM-5.3-Flash, Qwen3.6-35B-A3B, Qwen3-30B-A3B, MiMo-V2.6-Flash, Solar-Open2-250B, Nemotron 3 Nano 4B, Clef-Flash, lev | | `generate_qwen3moe` | `… --features gpu --release --bin generate_qwen3moe` | Qwen3.8-Flash-Next, Qwen3.6-35B-A3B, Qwen3-30B-A3B | | `generate_glm5next` | `… --features glm5next --release --bin generate_glm5next` | GLM-5.3-Flash | | `r8conv` | `cargo build --release -p bloomery-model --bin r8conv` | the V4.1 sidecar | diff --git a/docs/contrib/dgx-spark.md b/docs/contrib/dgx-spark.md index b7d637cb7..1a49f7ba9 100644 --- a/docs/contrib/dgx-spark.md +++ b/docs/contrib/dgx-spark.md @@ -56,7 +56,7 @@ the tested toolkit, answer 7) with LLVM 21.1.8's ARM64 release `llc` named by `C for example: ```sh -cargo oxide build --arch sm_86 -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,gemma-embedding2,hf --release --bin bloomery-serve +cargo oxide build --arch sm_86 -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve ``` The binary carries `sm_86` PTX, and the driver compiles it for cc 12.1 at the first start, about 6.5 s on the Arm cores @@ -73,7 +73,10 @@ load, as the first run did (section 3). Models by size, in the order to try them (`docs/models.md:12,31-34`, `docs/BUILD.md` table): Clef-Flash 4.8–9.7 GB, Qwen3-30B-A3B `Q4_K_M` 18.6 GB, Qwen3.6-35B-A3B `Q4_K_M` 21.2 GB; then Qwen3.8 `UD-Q4_K_XL` 111.3 GB and GLM-5.3-Flash `UD-Q2_K_XL` 101.3 GiB (the size chosen "to fit a DGX Spark", https://github.com/midagedev/rig-log/blob/main/docs/bloomery/plan-ledger.md#L359) if the first `MemAvailable` reading -allows. A 64 GB Spark (section 3; announced, not yet shipping) holds only the first three. +allows. NVIDIA's own `nvidia/Qwen3.8-Flash-Next-NVFP4` checkpoint (132.7 GB on disk) serves from its folder as +downloaded, `bloomery-serve -m `, with every routed expert on the GB10 and the PLE table read from the folder +on the NVMe tier (`docs/serving.md`, the seats). A 64 GB Spark (section 3; announced, not yet shipping) holds only the +first three. ## 3. The machine diff --git a/docs/serving.md b/docs/serving.md index e1e87c697..7492ab2c7 100644 --- a/docs/serving.md +++ b/docs/serving.md @@ -124,7 +124,8 @@ Your OpenAI client works unchanged. bloomery speaks llama-server's HTTP API and this server keeps the template's own default when a request names neither field (BUILD.md: thinking is off for V4.1). - **The files**: the GGUF uploads as downloaded, the same quantizations, the chat template read from the file, - and a tokenizer bit-identical to `llama-tokenize`. + and a tokenizer bit-identical to `llama-tokenize`. Qwen3.8 also serves NVIDIA's NVFP4 safetensors checkpoint as + downloaded (The seats, below). - **The flags**: `-m`/`--model-file`, `--hf [:]` (download, resume, sha256 check, never fetched twice; it also fetches the model's MTP draft when the repo publishes it, as unsloth's Qwen3.8-Flash-Next GGUFs do under `MTP/`, landing it beside the set, where the run picks it up), `--parallel/-np`, `--queue-depth`, @@ -184,6 +185,16 @@ Where it ends today: one model a server, sm_86+ GPUs, and the seats' own default ## The seats +The qwen38 seat serves a Qwen3.8 safetensors checkpoint folder as downloaded, with no conversion step, in a build with +the `checkpoint` feature (the release's has it; any other build refuses the folder by name): `-m` names the +folder that holds `model.safetensors.index.json`, and `--hf ` names a repo that holds such a checkpoint and no +GGUF set (a `:` on that repo is refused by name, as the checkpoint is the repo's one model). Every part keeps the +format the checkpoint stores it in. Checked with `nvidia/Qwen3.8-Flash-Next-NVFP4` on a DGX Spark's GB10, where it plans +the same bytes and gives the same greedy ids as the GGUF split made from it. The load holds every tensor on the stage +card and reads the PLE table from the folder on the NVMe tier, so a plan that puts a tensor elsewhere, a set +`BLOOMERY_RESIDENCY` other than `off`, and `BLOOMERY_DRAFT=mtp` are refused by name; unset, the MTP draft stays off, +as the folder's MTP layers are not read. + The nemotron seat serves Nemotron 3 Nano 4B (`--model nemotron -m PATH`) whole on one card, one slot: `--parallel` above 1, a placement of more than one card or with a tier card, and a draft lever are refused by name. The body has no prompt batch, so a prompt runs one decode step per token, at the decode rate. A Mamba-2 state cannot step back: a diff --git a/justfile b/justfile index 04e76c7c2..459cd5f95 100644 --- a/justfile +++ b/justfile @@ -48,7 +48,7 @@ lint: # bin check), and every recipe that builds these bins builds them with `vision`, so only this check compiles the # `cfg(not(feature = "vision"))` branches. A cargo check writes no executable, so it replaces no bin another lane runs. mac-combos runs this shape. check-no-vision: - ./tools/box.sh 'cargo check -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,gemma-embedding2,hf --bin bloomery-serve --bin bloomery-serve-ds41 --bin gate_glm5next_e2e' + ./tools/box.sh 'cargo check -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --bin bloomery-serve --bin bloomery-serve-ds41 --bin gate_glm5next_e2e' # fmt는 맥에서 돈다. box.sh의 rsync가 단방향이라 박스에서 포맷하면 결과가 돌아오지 # 않고 다음 명령에 덮여 사라진다(2026-09-19에 그렇게 한 번 날렸다). cargo fmt는 컴파일을 @@ -390,7 +390,7 @@ gate-gpu-solar2-twocard: [group('solo-real')] [group('v41-load')] gate-gpu-solar2-serve: - BLOOMERY_MODEL=solar-open2 ./tools/box.sh 'export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf --release --bin bloomery-serve --bin gate_solar2_serve && D=target/solar2-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=600 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_solar2_serve --dir $D' + BLOOMERY_MODEL=solar-open2 ./tools/box.sh 'export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_solar2_serve && D=target/solar2-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=600 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_solar2_serve --dir $D' # The nemotron seat of bloomery-serve on Nemotron 3 Nano 4B Q4_K_M (gate_nemotron_serve's doc): `--plan` with no flag names its # default context (the ladder's rung, held by the ctx line and the rung above refused) and exits before any load, `--parallel 2` is @@ -399,14 +399,14 @@ gate-gpu-solar2-serve: # server's template renders), serves that turn on /v1/chat/completions (usage counts its ids), resends an edited prompt as a # fresh run that keeps nothing and an extended one that keeps the held positions. One card. gate-gpu-nemotron-serve: - BLOOMERY_MODEL=nemotron_h ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf --release --bin bloomery-serve --bin gate_nemotron_serve && D=target/nemotron-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_nemotron_serve --dir $D' + BLOOMERY_MODEL=nemotron_h ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_nemotron_serve && D=target/nemotron-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_nemotron_serve --dir $D' # The same seat gate on the moe body (gate_nemotron_serve's doc, every clause one path over both # bodies): the Bolt fixture under the fixture tier (nemotron_h_moe, the fixture's four layers, any # card), the arch word and the layer count read from the file, the session's prompt call the body's # batched `GpuModel::prefill` (several ids; one id is a step). One card. gate-gpu-nemotron-h-moe-serve: - BLOOMERY_TIER=fixture BLOOMERY_MODEL=nemotron_h_moe ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf --release --bin bloomery-serve --bin gate_nemotron_serve && D=target/nemotron-h-moe-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_nemotron_serve --dir $D' + BLOOMERY_TIER=fixture BLOOMERY_MODEL=nemotron_h_moe ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_nemotron_serve && D=target/nemotron-h-moe-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_nemotron_serve --dir $D' # GLM-5.3-Flash 라우터(288개 중 8개, sigmoid, ×2.5 — `router::glm5next`)를 모델 없이 합성 입력으로 호스트 규칙과 맞춘다. # 규칙마다 절이 하나다: logits·ids·weights·batch는 비트, scores는 sigmoid 밴드, ties·guard·fault·graph는 명시한 값. @@ -1369,7 +1369,7 @@ gate-gpu-qwen35moe-media: [group('solo')] [group('v41-load')] gate-gpu-qwen38-media: - BLOOMERY_MODEL=qwen4exp ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-qwen38-media && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf --release --bin bloomery-serve --bin gate_qwen38_media && D=target/qwen38-media-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=2400 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_qwen38_media --dir $D' + BLOOMERY_MODEL=qwen4exp ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-qwen38-media && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_qwen38_media && D=target/qwen38-media-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=2400 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_qwen38_media --dir $D' # The decide seat of bloomery-serve (gate_clef_serve's doc): Clef-Flash at bartowski's Q5_K_M (6.9 GB, the # `qwen35` file of rev d7f376ea) with the release's head; the six refusals (a qwen35 file with no head, --model @@ -1387,7 +1387,7 @@ gate-gpu-qwen38-media: # adds the clauses that start the seat from the hub with no projector flag (it fetches the repo's own projector and # serves images) and with --no-mmproj (it serves none): a network read. Either card. gate-gpu-clef-serve *ARGS: - BLOOMERY_MODEL=qwen35 ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf --release --bin bloomery-serve --bin gate_clef_serve && D=target/clef-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_clef_serve --model /models/clef-flash/Cloudflare_clef-flash-Q5_K_M.gguf --head /models/clef-flash/hf/joint_head.safetensors --clef-model /root/models/clef-flash/Cloudflare_clef-flash-Q5_K_M.gguf --image-model /models/clef-flash/Cloudflare_clef-flash-Q8_0.gguf --mmproj /root/models/clef-flash/mmproj-Cloudflare_clef-flash-bf16.gguf --dir $D {{ARGS}}' + BLOOMERY_MODEL=qwen35 ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_clef_serve && D=target/clef-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_clef_serve --model /models/clef-flash/Cloudflare_clef-flash-Q5_K_M.gguf --head /models/clef-flash/hf/joint_head.safetensors --clef-model /root/models/clef-flash/Cloudflare_clef-flash-Q5_K_M.gguf --image-model /models/clef-flash/Cloudflare_clef-flash-Q8_0.gguf --mmproj /root/models/clef-flash/mmproj-Cloudflare_clef-flash-bf16.gguf --dir $D {{ARGS}}' # The decide seat of bloomery-serve on lev (gate_lev_serve's doc): ggml-org/lev-GGUF's Q4_K_M (3.0 GB, a `qwen35` # file with `qwen35.decision.type = lev`, fetched into /root/models/lev-4b/) with no head flag; five refusals before @@ -1397,7 +1397,7 @@ gate-gpu-clef-serve *ARGS: # prompts' ids; ARGS `--hf ggml-org/lev-GGUF:Q4_K_M` adds the clause that serves the file from the hub, no head flag, and # answers the first request as the `-m` server does (a network read). Either card. gate-gpu-lev-serve *ARGS: - BLOOMERY_MODEL=qwen35 ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf --release --bin bloomery-serve --bin gate_lev_serve && D=target/lev-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_lev_serve --model /root/models/lev-4b/lev-Q4_K_M.gguf --head /models/clef-flash/hf/joint_head.safetensors --ref $BLOOMERY_DATA/lev/4b/ref --dir $D {{ARGS}}' + BLOOMERY_MODEL=qwen35 ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_lev_serve && D=target/lev-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_lev_serve --model /root/models/lev-4b/lev-Q4_K_M.gguf --head /models/clef-flash/hf/joint_head.safetensors --ref $BLOOMERY_DATA/lev/4b/ref --dir $D {{ARGS}}' # clef_hidden on the box (ARGS as its doc: --model, --ids, --out, ...): a qwen35 file's prompt-only pass, every # position's final-norm hidden state to a file and a `clef hidden` record with the call's functional wall. @@ -1435,9 +1435,10 @@ gate-gpu-qwen4exp-iqleg: # router's flips named, and its last argmax; the batch set by one ubatch and D3K's step after its prefill by ubatches hold # the oracle's argmax (D3K's a named tie allowed); the distances to the other form's sets are recorded. The bin's unit # test, which holds the choice of sets to the split's dense form, runs first. BLOOMERY_REF_MODEL moves the split from -# the model folder below, including to prepare's stored-form output. Real-only: the fixture holds no NVFP4 stack. +# the model folder below, including to prepare's stored-form output, or names the checkpoint folder itself, which the +# load reads as stored (no prepare). Real-only: the fixture holds no NVFP4 stack. gate-gpu-qwen4exp-nvfp4: - BLOOMERY_REF_MODEL="${BLOOMERY_REF_MODEL:-/models/Qwen3.8-Flash-Next-NVFP4/Qwen3.8-Flash-Next-NVFP4-00001-of-00004.gguf}" BLOOMERY_MODEL=qwen4exp ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-qwen4exp-nvfp4 && bash tools/gate.sh --oxide -p bloomery-gpu-gates --release --features gpu --bin gate_qwen4exp_nvfp4 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin gate_qwen4exp_nvfp4 && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_qwen4exp_nvfp4' + BLOOMERY_REF_MODEL="${BLOOMERY_REF_MODEL:-/models/Qwen3.8-Flash-Next-NVFP4/Qwen3.8-Flash-Next-NVFP4-00001-of-00004.gguf}" BLOOMERY_MODEL=qwen4exp ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-qwen4exp-nvfp4 && bash tools/gate.sh --oxide -p bloomery-gpu-gates --release --features gpu,checkpoint --bin gate_qwen4exp_nvfp4 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu,checkpoint --release --bin gate_qwen4exp_nvfp4 && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_qwen4exp_nvfp4' # Qwen3.8's MTP draft program on the 3090 beside the target: first, before any walk, the draft's load — its weights, # store and row map hold its plan's bytes, its token_embd and output are the target's own buffers at their addresses, @@ -1528,7 +1529,7 @@ gate-gpu-minimaxm3-prefill: [group('solo')] [group('v41-load')] gate-gpu-minimaxm3-serve: - BLOOMERY_MODEL=minimax-m3 ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-minimaxm3-serve && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf --release --bin bloomery-serve --bin gate_minimaxm3_serve && D=target/minimaxm3-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1800 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_minimaxm3_serve --dir $D' + BLOOMERY_MODEL=minimax-m3 ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-minimaxm3-serve && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_minimaxm3_serve && D=target/minimaxm3-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1800 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_minimaxm3_serve --dir $D' # Head-256 pack-one prefill, whole inputs and symmetric windows, against the f64 attention band. gate-gpu-gemma-embedding-head256: @@ -1538,7 +1539,7 @@ gate-gpu-gemma-embedding-head256: # greedy tokens with embedding calls in flight, and named refusals; the runner holds one card. [group('pool')] gate-gpu-gemma-embedding2-serve *ARGS: - BLOOMERY_MODEL=qwen3moe ./tools/box.sh 'bash tools/gate.sh -p bloomery-gpu-gates --features hf --test sentence_prompts && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf --release --bin bloomery-serve --bin gate_gemma_embedding2_serve && D=target/embedding2-serve-gate-$(date +%s)-$$ && mkdir -p "$D" && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_serve --dir "$D" {{ARGS}} && if [ -z "{{ARGS}}" ]; then BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_serve --dir "$D/audio" --audio; fi' + BLOOMERY_MODEL=qwen3moe ./tools/box.sh 'bash tools/gate.sh -p bloomery-gpu-gates --features hf --test sentence_prompts && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_gemma_embedding2_serve && D=target/embedding2-serve-gate-$(date +%s)-$$ && mkdir -p "$D" && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_serve --dir "$D" {{ARGS}} && if [ -z "{{ARGS}}" ]; then BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_serve --dir "$D/audio" --audio; fi' # EmbeddingGemma 2's encoder (gpu-gemma-embedding2) end to end on the Q8_0 file the sets are dumped from # (refset's gemma_embedding2::MODEL, held to its sha256; the sets' `# model` line names the same path): every @@ -1550,7 +1551,7 @@ gate-gpu-gemma-embedding-e2e: BLOOMERY_MODEL=gemma-embedding ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gemma-embedding2,vision,deepseek41 --release --bin gate_gemma_embedding2_e2e && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_e2e --v1' gate-gpu-gemma-embedding-serve: - BLOOMERY_MODEL=qwen3moe ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf --release --bin bloomery-serve --bin gate_gemma_embedding2_serve && D=target/embedding-serve-gate-$(date +%s)-$$ && mkdir -p "$D" && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_serve --v1 --dir "$D"' + BLOOMERY_MODEL=qwen3moe ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_gemma_embedding2_serve && D=target/embedding-serve-gate-$(date +%s)-$$ && mkdir -p "$D" && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_serve --v1 --dir "$D"' gate-gpu-gemma-embedding2-e2e *ARGS: ./tools/box.sh 'bash tools/gate.sh --oxide --release -p bloomery-gpu-gemma-embedding2 --lib -- --include-ignored' @@ -2160,7 +2161,7 @@ gate-q38-serve-rules: # gate lock. [group('host')] gate-serve-draft-rules: - ./tools/box.sh 'bash tools/gate.sh --oxide -p bloomery-gpu-gates --release --features glm5next,clef,mimo2,solar2,minimaxm3,gemma-embedding2,hf --bin bloomery-serve --bin bloomery-serve-ds41' + ./tools/box.sh 'bash tools/gate.sh --oxide -p bloomery-gpu-gates --release --features glm5next,clef,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --bin bloomery-serve --bin bloomery-serve-ds41' # The V4.1 binaries' `--place` word (`generate::Place`, behind the deepseek41 feature like bind): each alias and its card # list (`a6000+3090` is `bp`) planning the machine its plan function makes, a list word planning its own cards, and every @@ -2635,7 +2636,7 @@ gate-gpu-qwen38-serve: # _decode_*): the qwen3moe file's --mmproj refusal and, on the Qwen3.6 file, the seat with its encoder. [group('pool')] gate-gpu-qwen3-serve: - BLOOMERY_MODEL=qwen3moe ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf --release --bin bloomery-serve --bin gate_qwen3_serve && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin generate_qwen3moe && D=target/qwen3-serve-gate && rm -rf $D && mkdir -p $D && Q36=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen35moe.sh) && test -n "$Q36" && Q38D=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen38d.sh) && test -n "$Q38D" && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_qwen3_serve --model "$BLOOMERY_REF_MODEL" --model "$Q36" --model "$Q38D" --dir $D --images' + BLOOMERY_MODEL=qwen3moe ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_qwen3_serve && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin generate_qwen3moe && D=target/qwen3-serve-gate && rm -rf $D && mkdir -p $D && Q36=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen35moe.sh) && test -n "$Q36" && Q38D=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen38d.sh) && test -n "$Q38D" && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_qwen3_serve --model "$BLOOMERY_REF_MODEL" --model "$Q36" --model "$Q38D" --dir $D --images' # bloomery-serve-qwen38 on the box, for a person to attach a client to (toktape records from it): the A6000 by # default (SERVE_PLACE=gate for the 3090), the port 8080 unless SERVE_PORT, a four-hour gate bound unless @@ -2754,7 +2755,7 @@ depth-gpu-glm5next *ARMS: # BLOOMERY_BOX_ENV=BLOOMERY_DRY=1 nothing is built and no lease is taken: the plan and each binary's state. depth-serve *ARMS: @[ -n "${BLOOMERY_MODEL:-}" ] || { echo "depth-serve: BLOOMERY_MODEL is unset: name the model profile (tools/ref/models/.sh), e.g. BLOOMERY_MODEL=mimo2 just depth-serve :" >&2; exit 64; } - ./tools/box.sh "${BLOOMERY_AB_ROUNDS:+export BLOOMERY_AB_ROUNDS=$BLOOMERY_AB_ROUNDS && }"'{{precheck}} && { here=$(basename "$PWD"); ours=; for a in {{ARMS}}; do t=${a#env=*:}; [ "${t%%:*}" != "$here" ] || ours=1; done; if [ -n "$ours" ] && [ -z "${BLOOMERY_DRY:-}" ]; then cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf --release --bin bloomery-serve; fi; } && bash tools/ref/depth-serve.sh {{ARMS}}' + ./tools/box.sh "${BLOOMERY_AB_ROUNDS:+export BLOOMERY_AB_ROUNDS=$BLOOMERY_AB_ROUNDS && }"'{{precheck}} && { here=$(basename "$PWD"); ours=; for a in {{ARMS}}; do t=${a#env=*:}; [ "${t%%:*}" != "$here" ] || ours=1; done; if [ -n "$ours" ] && [ -z "${BLOOMERY_DRY:-}" ]; then cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve; fi; } && bash tools/ref/depth-serve.sh {{ARMS}}' # The schemajump against Clef-Flash sitting: the Clef decide seat and a Qwen3.6 seat, both bloomery-serve, one lease on # the A6000, the arms alternating request by request (the design's §4; the cards are docs/cards/sj-*.card). Two windows @@ -2775,7 +2776,7 @@ depth-serve *ARMS: # bloomery-serve-ds41 picks, with SJ_DS41_MODEL; this recipe builds that binary when the windows name it) and SJ_SUITES # other suite files (`tools/ref/clef/suite-hard.jsonl`, scored against its `.gold.jsonl`); the header of the runner has both. sj-vs-clef: - ./tools/box.sh "${BLOOMERY_AB_ROUNDS:+export BLOOMERY_AB_ROUNDS=$BLOOMERY_AB_ROUNDS && }"'{{precheck}} && if [ -z "${BLOOMERY_DRY:-}" ]; then case " $(printf %s "${SJ_WINDOWS:-qwen:on qwen:off}" | tr , " ") " in *" qwen:"*) cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf --release --bin bloomery-serve;; esac && case " $(printf %s "${SJ_WINDOWS:-}" | tr , " ") " in *" ds41:"*) cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features deepseek41,vision,gemma-embedding2,hf --release --bin bloomery-serve-ds41;; esac; fi && bash tools/ref/sj-vs-clef.sh' + ./tools/box.sh "${BLOOMERY_AB_ROUNDS:+export BLOOMERY_AB_ROUNDS=$BLOOMERY_AB_ROUNDS && }"'{{precheck}} && if [ -z "${BLOOMERY_DRY:-}" ]; then case " $(printf %s "${SJ_WINDOWS:-qwen:on qwen:off}" | tr , " ") " in *" qwen:"*) cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve;; esac && case " $(printf %s "${SJ_WINDOWS:-}" | tr , " ") " in *" ds41:"*) cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features deepseek41,vision,gemma-embedding2,hf --release --bin bloomery-serve-ds41;; esac; fi && bash tools/ref/sj-vs-clef.sh' # Qwen3-30B-A3B decode-step kernel timeline (nsys, A6000, under the lease, lead-only): generate_qwen3moe's seed form at each # depth (default 6 4096), one prefill pass + BLOOMERY_NSYS_N - 1 replays, the cache height depth-qwen3moe.sh uses. The @@ -3330,7 +3331,7 @@ gate-gpu-glm5next-residency: [group('solo')] [group('v41-load')] gate-gpu-glm5next-serve: - BLOOMERY_MODEL=glm5next BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-glm5next-serve && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,hf,glm5next-hf --release --bin generate_glm5next && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf --release --bin bloomery-serve --bin gate_glm5next_serve && D=target/glm-serve-gate && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_glm5next_serve --arm plain --dir $D/plain && bash tools/gpu-gate.sh gate_glm5next_serve --arm drafted --dir $D/drafted' + BLOOMERY_MODEL=glm5next BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-glm5next-serve && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,hf,glm5next-hf --release --bin generate_glm5next && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_glm5next_serve && D=target/glm-serve-gate && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_glm5next_serve --arm plain --dir $D/plain && bash tools/gpu-gate.sh gate_glm5next_serve --arm drafted --dir $D/drafted' # The GLM seat's image input beside its draft (gate_glm5next_serve --arm media): one server of --parallel 2 under # BLOOMERY_DRAFT=mtp BLOOMERY_RESIDENCY=off with --mmproj, the GLM-5.3 projector, then an image chat on one slot and a @@ -3345,7 +3346,7 @@ gate-gpu-glm5next-serve: [group('solo')] [group('v41-load')] weekly-gpu-glm5next-media-serve: - BLOOMERY_MODEL=glm5next BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh weekly-gpu-glm5next-media-serve && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf --release --bin bloomery-serve --bin gate_glm5next_serve && D=target/glm-media-gate && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_glm5next_serve --arm media --dir $D --mmproj /models/mmproj/unsloth--GLM-5.3-Flash-GGUF/mmproj-BF16.gguf' + BLOOMERY_MODEL=glm5next BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh weekly-gpu-glm5next-media-serve && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_glm5next_serve && D=target/glm-media-gate && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_glm5next_serve --arm media --dir $D --mmproj /models/mmproj/unsloth--GLM-5.3-Flash-GGUF/mmproj-BF16.gguf' # The shipped `bloomery-serve` started once per image flag on the GLM-5.3 fixture (gate_serve_start's header has the clauses): (s1) # `-m ` with the serving defaults and one text chat; (s2) `--place a --mmproj `, the tower's @@ -3363,7 +3364,7 @@ weekly-gpu-glm5next-media-serve: # 8.6 GB set and the 1.2 GB projector in threads beside the first two starts: 53 to 98 s warm; the bound is the runner's 900 s. [group('solo')] gate-gpu-serve-start-glm: - BLOOMERY_MODEL=glm5next BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/fixture-only.sh gate-gpu-serve-start-glm && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf --release --bin bloomery-serve --bin gate_serve_start && D=target/serve-start-gate/glm && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_serve_start --family glm --dir $D' + BLOOMERY_MODEL=glm5next BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/fixture-only.sh gate-gpu-serve-start-glm && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_serve_start && D=target/serve-start-gate/glm && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_serve_start --family glm --dir $D' # The same gate on the V4.1 fixture (the file's architecture picks the seat, which has no `--plan`, so its plan is read from each # start's own log). The projector is the `# mmproj` line of the V4.1 vision set ($BLOOMERY_DATA/ref-vision/deepseek41v, `just dump-ref-vision`), @@ -3374,7 +3375,7 @@ gate-gpu-serve-start-glm: # beside the first two starts, a warm build: 72 to 132 s warm; the bound is the runner's 900 s. [group('solo')] gate-gpu-serve-start-ds41: - BLOOMERY_MODEL=deepseek41 BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/fixture-only.sh gate-gpu-serve-start-ds41 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf --release --bin bloomery-serve --bin gate_serve_start && D=target/serve-start-gate/ds41 && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_serve_start --family ds41 --dir $D' + BLOOMERY_MODEL=deepseek41 BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/fixture-only.sh gate-gpu-serve-start-ds41 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_serve_start && D=target/serve-start-gate/ds41 && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_serve_start --family ds41 --dir $D' # The MiMo seat of `bloomery-serve` (gate_mimo2_serve): the refusals before any load (--plan's default context, the one # slot, the tier card), one server at --ctx 4160 --parallel 1 holding its load and listening records, the three prompts' @@ -3391,7 +3392,7 @@ gate-gpu-serve-start-ds41: [group('solo')] [group('v41-load')] weekly-gpu-mimo2-serve: - BLOOMERY_MODEL=mimo2 BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh weekly-gpu-mimo2-serve && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf --release --bin bloomery-serve --bin gate_mimo2_serve && D=target/mimo2-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1680 bash tools/gpu-gate.sh gate_mimo2_serve --dir $D' + BLOOMERY_MODEL=mimo2 BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh weekly-gpu-mimo2-serve && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_mimo2_serve && D=target/mimo2-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1680 bash tools/gpu-gate.sh gate_mimo2_serve --dir $D' # glm5next decode CLI, functional run (no timing): generate_glm5next feeds --tokens one step per id, then greedy -n # tokens. The gate placement on the 3090 unless --place a (and BLOOMERY_CARD=a6000). 3090, gate lock. diff --git a/tools/bloomery/schema/bloomery-serve-ds41.jsonl b/tools/bloomery/schema/bloomery-serve-ds41.jsonl index 8bc3e8447..f3545d634 100644 --- a/tools/bloomery/schema/bloomery-serve-ds41.jsonl +++ b/tools/bloomery/schema/bloomery-serve-ds41.jsonl @@ -1,10 +1,11 @@ -{"records":1,"bin":"bloomery-serve-ds41","kinds":40} +{"records":1,"bin":"bloomery-serve-ds41","kinds":41} {"kind":"embedding_seat","head":"embedding_seat","doc":"The embedding model as the plan seats it before loading: its architecture, card, total card bytes, context bound and stage-card reserve (zero off the plan).","parts":[{"key":"arch","ty":"word","unit":""},{"key":"card","ty":"word","unit":""},{"key":"seat","ty":"word","unit":""},{"key":"bytes","ty":"u64","unit":"B"},{"key":"reserve","ty":"u64","unit":"B"},{"key":"ctx","ty":"u64","unit":"positions"}]} {"kind":"hf_offline","head":"hf offline","doc":"A `--hf` start with no listing — it failed at the network (no host, no connection, a timeout, no TLS, no reply), the hub answered busy or down (HTTP 429 or 5xx), or HF_HUB_OFFLINE asked for none: the cache's one verified set the quant names is used, its files `offline-cached`, and the reason.","parts":[{"key":"repo","ty":"word","unit":""},{"key":"reason","ty":"text","unit":""}]} {"kind":"hf_file","head":"hf file","doc":"A file of a `--hf` fetch: its path in the repo, its bytes, what the cache held (cached: checked, nothing fetched; fetch: from byte 0; resume: a partial file fetched from its end; stale: a cached file whose content is not the listing's, fetched again; offline-cached: a verified file used with no listing, after an `hf offline` line) and the byte the fetch starts at.","parts":[{"key":"file","ty":"word","unit":""},{"key":"bytes","ty":"u64","unit":"B"},{"key":"state","ty":"word","unit":""},{"key":"from","ty":"u64","unit":"B"}]} {"kind":"hf_progress","head":"hf progress","doc":"A running `--hf` download's bytes on disk, every few seconds.","parts":[{"key":"file","ty":"word","unit":""},{"key":"have","ty":"u64","unit":"B"},{"key":"of","ty":"u64","unit":"B"}]} {"kind":"hf_done","head":"hf done","doc":"A `--hf` file checked against the listing: the bytes this run fetched, the check (the LFS sha256, or the git blob sha1 of a file stored in git) and its digest.","parts":[{"key":"file","ty":"word","unit":""},{"key":"fetched_bytes","ty":"u64","unit":"B"},{"key":"check","ty":"word","unit":""},{"key":"digest","ty":"word","unit":""}]} {"kind":"hf_set","head":"hf set","doc":"The GGUF set `--hf [:]` picked in the repo's listing: its name, its tag, its files and their bytes.","parts":[{"key":"repo","ty":"word","unit":""},{"key":"set","ty":"word","unit":""},{"key":"tag","ty":"word","unit":""},{"key":"files","ty":"u64","unit":""},{"key":"bytes","ty":"u64","unit":"B"}]} +{"kind":"hf_checkpoint","head":"hf checkpoint","doc":"The safetensors checkpoint `--hf ` resolved to, the repo holding no GGUF set: the checkpoint folder is served as stored, and its files and their bytes are fetched.","parts":[{"key":"repo","ty":"word","unit":""},{"key":"files","ty":"u64","unit":""},{"key":"bytes","ty":"u64","unit":"B"}]} {"kind":"hf_draft_skip","head":"hf draft skipped","doc":"The `--hf` repo's MTP draft file, not fetched: the set picked cannot run it, and why, as the plan's own rule (`PlanInputs::mtp_borrows`) says it.","parts":[{"key":"file","ty":"word","unit":""},{"key":"reason","ty":"text","unit":""}]} {"kind":"place_unset","head":"place unset","doc":"The placement a serving seat runs by, after the levers' records and before the ctx line and the plan, and why: set (--place, its word as the plan line prints it), or unset and the common rule's choice on the cards — one card, the offer itself (one card); two cards, a when the body serves no tier card (two cards, the body serves no tier card) or no sitting has shown the tier not slower (two cards, no sitting has shown the tier not slower), and else by the plan of bp (the largest the stage, the next-largest its expert tier): kept when its tier holds at least the break-even and more than no expert (two cards, tier at or past the break-even), else a (two cards, tier under the break-even; two cards, the tier holds no expert; two cards, the tier's plan refused, the refusal on stderr). tier_experts is the experts the tier cards hold in that plan of bp, only where the rule asked the plan for them (never under --place, on one card, or on a refused plan); break_even and basis name the family's rule, basis the card file that decides it, whenever the family has them.","parts":[{"key":"place","ty":"word","unit":""},{"key":"why","ty":"text","unit":""},{"key":"tier_experts","ty":"u64","unit":"experts","opt":true},{"key":"break_even","ty":"u64","unit":"experts","opt":true},{"key":"basis","ty":"word","unit":"","opt":true}]} {"kind":"plan","head":"plan","doc":"The placement the engine is about to load by: its card, context, where the experts sit, the routed-expert bytes the plan pages through the NVMe tier (paged), the RAM arena they page through (arena) and the slots it holds (arena_slots), 0 and 0 and 0 on a plan that pages nothing, the host room's reading the plan was made at (room) and the plan room it was held to (plan_room, the reading less the drift margin) when it was made at one, and each plan card's device (role:name:ordinal:usable bytes; by-name for a card a census-free plan opens by its name) under the CUDA enumeration order in force, with the card's free bytes at plan time when the census read them.","parts":[{"key":"place","ty":"word","unit":""},{"key":"card","ty":"word","unit":""},{"key":"ctx_max","ty":"u64","unit":"positions"},{"key":"card_experts","ty":"u64","unit":"experts"},{"lit":" ("},{"pos":"card_expert_bytes","ty":"u64","unit":"B"},{"lit":" B)"},{"key":"host_experts","ty":"u64","unit":"experts"},{"lit":" ("},{"pos":"host_expert_bytes","ty":"u64","unit":"B"},{"lit":" B)"},{"key":"host_shadow","ty":"u64","unit":"B"},{"lit":" B"},{"key":"n_l","ty":"word","unit":"experts"},{"lit":" on "},{"pos":"n_l_layers","ty":"u64","unit":"layers"},{"lit":" layers"},{"key":"card_budget","ty":"word","unit":"B"},{"key":"paged","ty":"u64","unit":"B"},{"key":"arena","ty":"u64","unit":"B"},{"key":"arena_slots","ty":"u64","unit":"slots"},{"key":"room","ty":"u64","unit":"B","opt":true},{"key":"plan_room","ty":"u64","unit":"B","opt":true},{"key":"card_free","ty":"u64","unit":"B","opt":true},{"key":"devices","ty":"csv","unit":"","opt":true},{"key":"cuda_order","ty":"word","unit":"","opt":true}]} diff --git a/tools/bloomery/schema/bloomery-serve-qwen38.jsonl b/tools/bloomery/schema/bloomery-serve-qwen38.jsonl index 47c1e283a..21e1ba448 100644 --- a/tools/bloomery/schema/bloomery-serve-qwen38.jsonl +++ b/tools/bloomery/schema/bloomery-serve-qwen38.jsonl @@ -1,10 +1,11 @@ -{"records":1,"bin":"bloomery-serve-qwen38","kinds":34} +{"records":1,"bin":"bloomery-serve-qwen38","kinds":35} {"kind":"embedding_seat","head":"embedding_seat","doc":"The embedding model as the plan seats it before loading: its architecture, card, total card bytes, context bound and stage-card reserve (zero off the plan).","parts":[{"key":"arch","ty":"word","unit":""},{"key":"card","ty":"word","unit":""},{"key":"seat","ty":"word","unit":""},{"key":"bytes","ty":"u64","unit":"B"},{"key":"reserve","ty":"u64","unit":"B"},{"key":"ctx","ty":"u64","unit":"positions"}]} {"kind":"hf_offline","head":"hf offline","doc":"A `--hf` start with no listing — it failed at the network (no host, no connection, a timeout, no TLS, no reply), the hub answered busy or down (HTTP 429 or 5xx), or HF_HUB_OFFLINE asked for none: the cache's one verified set the quant names is used, its files `offline-cached`, and the reason.","parts":[{"key":"repo","ty":"word","unit":""},{"key":"reason","ty":"text","unit":""}]} {"kind":"hf_file","head":"hf file","doc":"A file of a `--hf` fetch: its path in the repo, its bytes, what the cache held (cached: checked, nothing fetched; fetch: from byte 0; resume: a partial file fetched from its end; stale: a cached file whose content is not the listing's, fetched again; offline-cached: a verified file used with no listing, after an `hf offline` line) and the byte the fetch starts at.","parts":[{"key":"file","ty":"word","unit":""},{"key":"bytes","ty":"u64","unit":"B"},{"key":"state","ty":"word","unit":""},{"key":"from","ty":"u64","unit":"B"}]} {"kind":"hf_progress","head":"hf progress","doc":"A running `--hf` download's bytes on disk, every few seconds.","parts":[{"key":"file","ty":"word","unit":""},{"key":"have","ty":"u64","unit":"B"},{"key":"of","ty":"u64","unit":"B"}]} {"kind":"hf_done","head":"hf done","doc":"A `--hf` file checked against the listing: the bytes this run fetched, the check (the LFS sha256, or the git blob sha1 of a file stored in git) and its digest.","parts":[{"key":"file","ty":"word","unit":""},{"key":"fetched_bytes","ty":"u64","unit":"B"},{"key":"check","ty":"word","unit":""},{"key":"digest","ty":"word","unit":""}]} {"kind":"hf_set","head":"hf set","doc":"The GGUF set `--hf [:]` picked in the repo's listing: its name, its tag, its files and their bytes.","parts":[{"key":"repo","ty":"word","unit":""},{"key":"set","ty":"word","unit":""},{"key":"tag","ty":"word","unit":""},{"key":"files","ty":"u64","unit":""},{"key":"bytes","ty":"u64","unit":"B"}]} +{"kind":"hf_checkpoint","head":"hf checkpoint","doc":"The safetensors checkpoint `--hf ` resolved to, the repo holding no GGUF set: the checkpoint folder is served as stored, and its files and their bytes are fetched.","parts":[{"key":"repo","ty":"word","unit":""},{"key":"files","ty":"u64","unit":""},{"key":"bytes","ty":"u64","unit":"B"}]} {"kind":"hf_draft_skip","head":"hf draft skipped","doc":"The `--hf` repo's MTP draft file, not fetched: the set picked cannot run it, and why, as the plan's own rule (`PlanInputs::mtp_borrows`) says it.","parts":[{"key":"file","ty":"word","unit":""},{"key":"reason","ty":"text","unit":""}]} {"kind":"plan38","head":"plan","doc":"A qwen3moe-family placement the engine is about to load by: its card, the architecture (a placed qwen3moe or qwen35moe file names it; a qwen4exp line does not), the expert rule (host or card), the context, where the routed experts sit (each layer's id prefix on the card, a placed line stating as n_l the fewest and the most one layer keeps; under plan (b′) the expert tier card and the next ids it holds), each plan card's device as the plan record names them, the card's free bytes at plan time when the census read them, the host room's reading the plan was made at (room) and the plan room it was held to (plan_room, the reading less the drift margin) when it was made at one, and — on the placed plan a run with --place unset took because the whole file did not fit the free bytes — why it did not; for a file with a row-gathered table (a qwen4exp file's PLE table), the tier the plan reads it from (rows, host or nvme) and the host room's reading that chose it (read).","parts":[{"key":"place","ty":"word","unit":""},{"key":"card","ty":"word","unit":""},{"key":"arch","ty":"word","unit":"","opt":true},{"key":"experts","ty":"word","unit":""},{"key":"ctx_max","ty":"u64","unit":"positions"},{"key":"host_experts","ty":"u64","unit":"experts"},{"key":"card_experts","ty":"u64","unit":"experts"},{"key":"n_l","ty":"word","unit":"experts","opt":true},{"key":"tier","ty":"word","unit":"","opt":true},{"key":"tier_experts","ty":"u64","unit":"experts","opt":true},{"key":"card_free","ty":"u64","unit":"B","opt":true},{"key":"room","ty":"u64","unit":"B","opt":true},{"key":"plan_room","ty":"u64","unit":"B","opt":true},{"key":"why","ty":"word","unit":"","opt":true},{"key":"rows","ty":"word","unit":"","opt":true},{"key":"read","ty":"word","unit":"","opt":true},{"key":"devices","ty":"csv","unit":"","opt":true},{"key":"cuda_order","ty":"word","unit":"","opt":true}]} {"kind":"need","head":"need","doc":"What the load about to start needs of this machine, printed with the plan: bytes, the host need the load checks against MemAvailable (the plan's host terms, the churn pool and the bytes held beside the plan, less the OS reserve), by page class (pinned: driver-pinned shadows, tier rows and a pinned arena; anon: an anonymous NVMe-tier arena; cache: file pages the kernel may reclaim; locked: file pages mlocked under BLOOMERY_HOST_LOCK, in place of the cache ones; unified: the cards' share of an integrated device's pool; they sum to bytes), os (the OS reserve left out of bytes), cache_ram (the serve's prompt cache budget in anonymous pages, beside bytes, 0 where there is none), lock (BLOOMERY_HOST_LOCK), cards (each plan card's name and the bytes the plan puts on it, stage cards then tiers), and the worker pool's placement: threads (participants, the calling thread among them), cpus (the cpu each of the threads - 1 workers is pinned to, empty for a pool whose pins failed), caller (the cpu the dispatcher's slot pins to under pin_caller, which only a binary that asks for it does) and allowed (every cpu the process may run on: the caller, the engine's threads and the helpers float over it). The line BLOOMERY_NEED_HOOK reads.","parts":[{"key":"bytes","ty":"u64","unit":"B"},{"key":"pinned","ty":"u64","unit":"B"},{"key":"anon","ty":"u64","unit":"B"},{"key":"cache","ty":"u64","unit":"B"},{"key":"locked","ty":"u64","unit":"B"},{"key":"unified","ty":"u64","unit":"B"},{"key":"os","ty":"u64","unit":"B"},{"key":"cache_ram","ty":"u64","unit":"B"},{"key":"lock","ty":"bool","unit":""},{"key":"cards","ty":"csv","unit":"name:B"},{"key":"threads","ty":"u64","unit":"participants"},{"key":"cpus","ty":"csv","unit":"cpu ids"},{"key":"caller","ty":"u64","unit":"cpu id"},{"key":"allowed","ty":"csv","unit":"cpu ids"}]} diff --git a/tools/recipes.py b/tools/recipes.py index 9815eebe0..392201182 100755 --- a/tools/recipes.py +++ b/tools/recipes.py @@ -2466,8 +2466,9 @@ def cmd_why(args: argparse.Namespace) -> int: KEY_REACH_PINS = { "bloomery-hf": "a Hugging Face fetch or checkpoint configuration reader: its commits move only the gates whose targets run its code " "(model-hf, gpu-gates under the `hf` feature, and gpu-glm5next under its `hf` feature for gate-gpu-glm5next-tp's folder test)", - "bloomery-model-hf": "the fixture fetch and GLM's checkpoint map: gpu-gates links it under the `glm5next-hf` feature, for generate_glm5next, " - "and gpu-glm5next under its `hf` feature, for gate-gpu-glm5next-tp's folder test", + "bloomery-model-hf": "the fixture fetch and the checkpoint maps: gpu-gates links it under the `checkpoint` feature, for a Qwen3.8 checkpoint " + "folder that bloomery-serve and gate_qwen4exp_nvfp4 open and for generate_glm5next (`glm5next-hf`), and gpu-glm5next under its `hf` " + "feature, for gate-gpu-glm5next-tp's folder test", } diff --git a/tools/ref/dense-logits.py b/tools/ref/dense-logits.py index 6b6bddb9b..e3c185588 100644 --- a/tools/ref/dense-logits.py +++ b/tools/ref/dense-logits.py @@ -234,7 +234,7 @@ def native_build(args): env = os.environ.copy() env["CARGO_TARGET_DIR"] = str(ROOT / "target") argv = ["cargo", "oxide", "build", "--arch", args.arch, "--", "-p", "bloomery-gpu-gates", - "--features", "glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf", "--release", + "--features", "glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint", "--release", "--bin", "generate_qwen3moe", "--bin", "bloomery-serve"] subprocess.run(argv, cwd=ROOT, env=env, check=True, timeout=1800) if clean_head(ROOT) != head: diff --git a/tools/release/build.sh b/tools/release/build.sh index 354ee574b..d48cd3277 100755 --- a/tools/release/build.sh +++ b/tools/release/build.sh @@ -191,7 +191,7 @@ else bin_args=() for b in "${BINS[@]}"; do bin_args+=(--bin "$b"); done # The commit goes to cargo alone: the serve build script reads it, and a running binary never does. - BLOOMERY_GIT_COMMIT="$COMMIT" cargo oxide build --arch "$ARCH" -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf --release "${bin_args[@]}" + BLOOMERY_GIT_COMMIT="$COMMIT" cargo oxide build --arch "$ARCH" -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release "${bin_args[@]}" fi fail=0 diff --git a/tools/unsafe-ratchet.txt b/tools/unsafe-ratchet.txt index 596b75d75..5aabb6afe 100644 --- a/tools/unsafe-ratchet.txt +++ b/tools/unsafe-ratchet.txt @@ -26,7 +26,9 @@ embed 0 0 0 # libc's pread into a buffer that may hold bytes never written (read_span_uninit, which the NVMe tier's pinned arena # reads into), one block, and read_span views its initialized bytes as that buffer, one block; each with its SAFETY # line. -engram 10 0 0 +# PIN(2026-10-22): engram 10 -> 11 blocks, new code, not old code relaxed: RunTable maps each file that holds a +# table's runs (a checkpoint folder's shards) read-only with memmap2's map, one block with its SAFETY line. +engram 11 0 0 engram-lab 0 0 0 gguf 7 2 0 # PIN(2026-10-01): gpu 1354 -> 1356, a new kernel entry, not old code relaxed: kda_delta_lanes (GLM's two-row @@ -552,10 +554,15 @@ metal 21 6 13 # compute call (ops.rs dispatch_ccd, over each run of a block) and the warm walk's prefetch (ops.rs warm_touch), each # with its SAFETY line, and the cut test's RowSet double (ccd_tests::Pairs, whose `unsafe fn compute` the trait # makes unsafe). -model 108 31 13 +# PIN(2026-10-22): model 108 -> 109 blocks, 31 -> 32 fns, new code, not old code relaxed: host_lock's +# release_file_pages, an unsafe fn whose caller states that the slice is a file's own read-only mapping, and the one +# block that hands that slice to FileMapping::new, with its SAFETY line. +model 109 32 13 # model-hf: the four blocks that left model (119 -> 115) with the fixture bin and its test, the page-fault reads (`getrusage`), # each with its SAFETY line; no new unsafe. -model-hf 0 0 0 +# PIN(2026-10-22): model-hf 0 -> 1 block, new code, not old code relaxed: Checkpoint::release calls +# release_file_pages on the shard mapping safetensors::Folder::mapped gives, with its SAFETY line. +model-hf 1 0 0 models 0 0 0 # PIN(2026-10-21): os 24 blocks, 5 fns, not old code relaxed: the host chain's OS calls moved here from threads, # gguf, engram and model (their pins come down), each beside its macOS body, each block with its SAFETY line.