gpu, model-hf, placement: read a checkpoint folder's parts in parallel within the host load budget

crates/gguf/src/quant.rs

@@ -908,10 +908,14 @@ pub fn fp4_pair(byte: u8) -> [i8; 2] {
 #[inline]
 #[must_use]
 pub fn nvfp4_group_to_block(g: &[u8; 8]) -> [u8; 8] {
-    std::array::from_fn(|j| {
-        let nib = |x: u8| (x >> (4 * (j % 2))) & 0x0f;
-        nib(g[j / 2]) | (nib(g[4 + j / 2]) << 4)
-    })
+    let spread = |half: [u8; 4]| {
+        let x = u64::from(u32::from_le_bytes(half));
+        let x = (x | x << 16) & 0x0000_ffff_0000_ffff;
+        let x = (x | x << 8) & 0x00ff_00ff_00ff_00ff;
+        (x | x << 4) & 0x0f0f_0f0f_0f0f_0f0f
+    };
+    let [a, b, c, d, e, f, h, i] = *g;
+    (spread([a, b, c, d]) | spread([e, f, h, i]) << 4).to_le_bytes()
 }
 
 /// The block scale of an MXFP4 block, `ggml_e8m0_to_fp32_half` (ggml-impl.h:40-45):

crates/gpu-gates/src/bin/bloomery_serve.rs

@@ -358,42 +358,43 @@ models (the --model word is optional with a model file: the file's architecture
     /// the seat words. A qwen35moe file (Qwen3.6) is the qwen3 seat's.
     fn seat_of_file(generative: bool) -> Result<(Model, String), GateError> {
         let path = ref_model_path()?;
-        let split = model_file::header(&path)?;
-        let (seat, arch) = match split.architecture() {
-            Some(a) if side_seat::row(a).is_some() => {
-                return Err(format!(
-                    "{} is a {a} file, an embedding model: serve it beside a chat model with \
+        model_file::with_header(&path, |split| {
+            let (seat, arch) = match split.architecture() {
+                Some(a) if side_seat::row(a).is_some() => {
+                    return Err(format!(
+                        "{} is a {a} file, an embedding model: serve it beside a chat model with \
                      --embd-model",
-                    path.display()
-                )
-                .into());
-            }
-            Some(a) if !generative && decide::ROWS.iter().any(|r| r.backbones.contains(&a)) => {
-                return Err(format!(
-                    "{}: {}",
-                    path.display(),
-                    serve::decide::no_head(a, decide::ROWS)
-                )
-                .into());
-            }
-            Some(a) => (
-                Model::ALL
-                    .iter()
-                    .copied()
-                    .find(|m| m.serves(a))
-                    .ok_or_else(|| no_seat(&path, &split, a))?,
-                a.to_owned(),
-            ),
-            None => {
-                return Err(format!(
-                    "{}: the first shard names no architecture; the seats are --model {}",
-                    path.display(),
-                    seat_words()
-                )
-                .into());
-            }
-        };
-        Ok((seat, arch))
+                        path.display()
+                    )
+                    .into());
+                }
+                Some(a) if !generative && decide::ROWS.iter().any(|r| r.backbones.contains(&a)) => {
+                    return Err(format!(
+                        "{}: {}",
+                        path.display(),
+                        serve::decide::no_head(a, decide::ROWS)
+                    )
+                    .into());
+                }
+                Some(a) => (
+                    Model::ALL
+                        .iter()
+                        .copied()
+                        .find(|m| m.serves(a))
+                        .ok_or_else(|| no_seat(&path, split, a))?,
+                    a.to_owned(),
+                ),
+                None => {
+                    return Err(format!(
+                        "{}: the first shard names no architecture; the seats are --model {}",
+                        path.display(),
+                        seat_words()
+                    )
+                    .into());
+                }
+            };
+            Ok((seat, arch))
+        })?
     }
 
     /// The refusal of a file of architecture `a` that no seat serves: what no
@@ -425,12 +426,13 @@ models (the --model word is optional with a model file: the file's architecture
     /// ([`ref_model_path`]) and its `<arch>.decision.type`, when it has one.
     fn file_facts() -> Result<(String, Option<String>), GateError> {
         let path = ref_model_path()?;
-        let split = model_file::header(&path)?;
-        let decision = bloomery_gpu_gates::model_file::decision_type(&split)?.map(str::to_owned);
-        Ok((
-            split.architecture().unwrap_or("<missing>").to_owned(),
-            decision,
-        ))
+        model_file::with_header(&path, |split| {
+            let decision = model_file::decision_type(split)?.map(str::to_owned);
+            Ok((
+                split.architecture().unwrap_or("<missing>").to_owned(),
+                decision,
+            ))
+        })?
     }
 
     /// The head the arguments name ([`serve::decide::pick`]) and the rest:

crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3_folder.rs

@@ -32,8 +32,7 @@ pub struct Opened {
 /// body does not read), is refused by name.
 #[cfg(feature = "checkpoint")]
 pub fn open(dir: &Path) -> Result<Opened, GateError> {
-    let f = model_hf::folder::open(dir)
-        .map_err(|e| format!("checkpoint folder {}: {e}", dir.display()))?;
+    let f = bloomery_gpu_gates::model_file::open_folder(dir)?;
     let arch = refset::arch::minimaxm3::ARCH;
     if f.header.architecture() != Some(arch) {
         return Err(format!(

crates/gpu-gates/src/bin/shared/serve_seats/qwen38_folder.rs

@@ -24,8 +24,7 @@ pub struct Opened {
 /// architecture is refused by name.
 #[cfg(feature = "checkpoint")]
 pub fn open(path: &Path) -> Result<Opened, GateError> {
-    let mut f = model_hf::folder::open(path)
-        .map_err(|e| format!("checkpoint folder {}: {e}", path.display()))?;
+    let mut f = bloomery_gpu_gates::model_file::open_folder(path)?;
     if f.header.architecture() != Some("qwen4exp") {
         return Err(format!(
             "checkpoint folder {}: a {} model, where this seat serves qwen4exp",

crates/gpu-gates/src/bind.rs

@@ -3597,6 +3597,7 @@ mod tests {
                 shadows: 0,
                 reserves: 0,
                 extra: 0,
+                read_ahead: 0,
                 arena: 0,
                 cards: 0,
                 os: 0,

crates/gpu-gates/src/model_file.rs

@@ -29,6 +29,47 @@ pub use fetch::{
 pub(crate) const NONE: &str = "no model file: pass -m PATH (the first shard, or a checkpoint folder) \
                                or --hf <repo>[:<quant>], or set BLOOMERY_REF_MODEL";
 
+#[cfg(feature = "checkpoint")]
+static KEPT: std::sync::Mutex<Option<(PathBuf, model_hf::folder::Opened)>> =
+    std::sync::Mutex::new(None);
+
+/// The folder at `path`, reusing the open whose header selected its server seat.
+#[cfg(feature = "checkpoint")]
+pub fn open_folder(path: &Path) -> Result<model_hf::folder::Opened, GateError> {
+    let kept = KEPT
+        .lock()
+        .unwrap_or_else(std::sync::PoisonError::into_inner)
+        .take();
+    match kept {
+        Some((p, opened)) if p == path => Ok(opened),
+        _ => read_folder(path),
+    }
+}
+
+#[cfg(feature = "checkpoint")]
+fn read_folder(path: &Path) -> Result<model_hf::folder::Opened, GateError> {
+    model_hf::folder::open(path)
+        .map_err(|e| format!("checkpoint folder {}: {e}", path.display()).into())
+}
+
+/// `f` of the model's header, keeping a checkpoint folder for its seat's open.
+pub fn with_header<R>(path: &Path, f: impl FnOnce(&Split) -> R) -> Result<R, GateError> {
+    #[cfg(feature = "checkpoint")]
+    if path.is_dir() {
+        let mut kept = KEPT
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        let opened = match kept.take() {
+            Some((p, opened)) if p == path => opened,
+            _ => read_folder(path)?,
+        };
+        let result = f(&opened.header);
+        *kept = Some((path.to_path_buf(), opened));
+        return Ok(result);
+    }
+    header(path).map(|header| f(&header))
+}
+
 /// The header of the model at `path`, which a server seats it by and `-ngl N` is compared with:
 /// its GGUF split's first shard, or a checkpoint folder's when `path` is a directory, which the
 /// map its config names reads (`model_hf::folder::open`), so the seat is chosen from the header's
@@ -43,9 +84,7 @@ pub fn header(path: &Path) -> Result<Split, GateError> {
 /// The header of the checkpoint folder `path`.
 #[cfg(feature = "checkpoint")]
 fn folder_header(path: &Path) -> Result<Split, GateError> {
-    model_hf::folder::open(path)
-        .map(|f| f.header)
-        .map_err(|e| format!("checkpoint folder {}: {e}", path.display()).into())
+    read_folder(path).map(|f| f.header)
 }
 
 /// A checkpoint folder in a build without the `checkpoint` feature, refused by name.

crates/gpu-glm5next/src/tp/mod.rs

@@ -620,7 +620,7 @@ impl Drop for Rank {
 /// ([`RankOf::take`]), in its type's card format.
 pub fn load(
     gpu: &Gpu,
-    source: &impl Source,
+    source: &(impl Source + Sync),
     at: RankOf,
     layers: usize,
     heads: usize,

crates/gpu/Cargo.toml

@@ -34,7 +34,7 @@ bloomery-verbs = { path = "../verbs" }
 # The expert tier's page shared with a tier node's process through a /dev/shm file (`graph::MappedHost`).
 memmap2.workspace = true
 libc.workspace = true
-# The huge-page advice for large filled host memory (`tensor::host_filled`).
+# The huge-page advice for large new host memory (`tensor::host_filled`, `upload::pack_planes`).
 bloomery-os.workspace = true
 # The pinned pool the hybrid host tier waits on and computes with (the CPU
 # engine dispatches its expert matmuls on the same pool).

crates/gpu/src/arch/qwen3moe/mtp38.rs

@@ -260,7 +260,7 @@ impl Mtp38 {
     pub fn open(
         gpu: &Gpu,
         target_w: &Weights,
-        (draft_file, parts): (&Split, Option<&crate::model::PartsLoad<'_>>),
+        (draft_file, parts): (&Split, Option<&dyn crate::model::PartsLoad>),
         mtp: &model::arch::qwen35moe::place::MtpInputs,
         plan: &model::arch::qwen35moe::place::MtpPlan<'_>,
         card_dontneed: bool,
@@ -309,7 +309,13 @@ impl Mtp38 {
         )?;
         let rows = file_rows(&plan.draft);
         let mut w = match parts {
-            Some(load) => load(stream, &rows, 0, card_dontneed)?,
+            Some(parts) => {
+                let available = model::placement::workstation::host_available()
+                    .map_err(|e| GpuError::plan(WHAT, e))?;
+                let (need, room) = parts.host_need(&rows, 0, 0, available)?;
+                need.check(available).map_err(|e| GpuError::plan(WHAT, e))?;
+                parts.load(stream, &rows, 0, card_dontneed, room)?
+            }
             None => Weights::load_placed(stream, draft_file, &rows, 0, card_dontneed)?,
         };
         for t in &mtp.tensors {

crates/gpu/src/folder.rs

@@ -11,23 +11,18 @@
 //! machine's source reads the experts where the folder stores them, through the located header
 //! (`crate::host::swap_source::FileSwap`).
 
-use cuda_core::CudaStream;
 use gguf::Split;
 use model::part::Source;
-use model::placement::Plan;
 
 use crate::GpuError;
 use crate::host::swap::Residency;
 use crate::model::CardBytes;
-use crate::parts::{InPlace, load_placed_in_place};
-use crate::weights::Weights;
+use crate::parts::{InPlace, PlacedSource};
 
 /// What the errors name.
 const WHAT: &str = "folder::Model::into_origin";
 
-/// The card's tensors of a plan from a part source ([`load_placed_in_place`]).
-type CardParts =
-    Box<dyn Fn(&CudaStream, &Plan<'_>, usize, bool) -> Result<Weights, GpuError> + Send>;
+type CardParts = Box<dyn crate::model::PartsLoad>;
 
 /// The model a load reads: a GGUF split, or a checkpoint folder ([`Folder`]). `X` is what the
 /// folder's architecture carries beside the folder's parts.
@@ -91,7 +86,7 @@ impl<X> From<Folder<X>> for Model<X> {
 /// `gguf::Split::header_only` for a folder whose host reads nothing where the folder stores it),
 /// the header its plan is made from, whose in-place tensors the host reads where the folder
 /// stores them ([`InPlace`]), and its card tensors through its part source
-/// ([`load_placed_in_place`]). `X` is the architecture's own addition to the load, `()` for
+/// ([`crate::parts::load_placed_in_place`]). `X` is the architecture's own addition to the load, `()` for
 /// none.
 pub struct Folder<X = ()> {
     header: Split,
@@ -103,25 +98,23 @@ impl<X> Folder<X> {
     /// The folder of header `header`, its card tensors read from `source`, with the
     /// architecture's `extra`. A load reads only its plan's tensors of the source, so the source
     /// may also hold another plan's (a draft's, which loads through [`Origin::bytes`] too).
-    pub fn with<S: Source + Send + 'static>(header: Split, source: S, extra: X) -> Folder<X> {
+    pub fn with<S: Source + Send + Sync + 'static>(
+        header: Split,
+        source: S,
+        extra: X,
+    ) -> Folder<X> {
         let in_place = InPlace::of(&header);
         Folder {
             header,
-            parts: Box::new(move |stream, plan, card, release| {
-                let selected = model::part::Filtered {
-                    source: &source,
-                    keep: |name: &str| plan.model.tensors.iter().any(|t| t.name == name),
-                };
-                load_placed_in_place(stream, &selected, plan, card, release, &in_place)
-            }),
+            parts: Box::new(PlacedSource { source, in_place }),
             extra,
         }
     }
 }
 
 impl Folder {
     /// The folder of header `header`, its card tensors read from `source`.
-    pub fn new<S: Source + Send + 'static>(header: Split, source: S) -> Folder {
+    pub fn new<S: Source + Send + Sync + 'static>(header: Split, source: S) -> Folder {
         Folder::with(header, source, ())
     }
 }
@@ -175,6 +168,10 @@ mod tests {
             Err(std::fmt::Error)
         }
 
+        fn read_bytes(&self, _: &Planned) -> Result<u64, Self::Error> {
+            Err(std::fmt::Error)
+        }
+
         fn read(&self, _: &Planned) -> Result<Part<'_>, Self::Error> {
             Err(std::fmt::Error)
         }

crates/gpu/src/host/node.rs

@@ -417,7 +417,12 @@ impl NodeOpen {
                     None => {
                         Weights::load_placed(gpu.stream(), file, plan, self.card, card_dontneed)?
                     }
-                    Some(load) => load(gpu.stream(), plan, self.card, card_dontneed)?,
+                    Some(load) => {
+                        let available =
+                            workstation::host_available().map_err(|e| GpuError::plan(WHAT, e))?;
+                        let room = load.host_bytes(plan, self.card, available)?;
+                        load.load(gpu.stream(), plan, self.card, card_dontneed, room)?
+                    }
                 };
                 let experts = (family.experts)(&gpu, file, &tensors, &w, &self)?;
                 (w, experts)