tests, tokenizer and probe: close ten open review follow-ups

crates/gpu-gates/src/bin/generate_qwen3moe.rs

@@ -391,6 +391,12 @@ fn dense_shortage_halves_the_default_context() {
     q3place::tests::dense_shortage_halves_the_default_context();
 }
 
+#[cfg(all(test, feature = "gpu"))]
+#[test]
+fn a_dense_header_plans_through_qwen35_on() {
+    q3place::tests::a_dense_header_plans_through_qwen35_on();
+}
+
 #[cfg(feature = "gpu")]
 #[path = "shared/qwen38_place.rs"]
 mod q38place;

crates/gpu-gates/src/bin/h2d_probe.rs

@@ -45,7 +45,17 @@
 //!   interval_ms=<ms> bytes=<b> GB/s=<v>`, bytes counted when their copy is
 //!   known complete; then one `h2d loop arm=<a> summary …` line.
 //!
-//! Every copy arm samples each of its cards'
+//! - `readback`: the other direction, card to host, into host memory each
+//!   call allocates, as a request path's destination: `--reps` (8) calls of
+//!   `--bytes` (32 MiB) into memory nothing has written
+//!   (`DeviceBuffer::to_host_vec`), then as many into memory the host writes
+//!   first (`bloomery_gpu::to_host`), each timed on the host clock with its
+//!   fill: `h2d arm=readback … dest=fresh|filled bytes= calls= first_ms=
+//!   median_ms=`. A card that reaches pageable memory through the host's page
+//!   tables (GB10's unified memory) pays each fresh page's first touch inside
+//!   the copy.
+//!
+//! Every copy arm but `readback` samples each of its cards'
 //! `current_link_speed`/`current_link_width` from sysfs every `SAMPLE_EVERY` while
 //! its copies are in flight (an idle link trains down) and prints `h2d link
 //! arm=<a> card= bdf= samples=<n> [<speed> x<width>: <count>, …]`.
@@ -56,6 +66,8 @@
 
 #[cfg(feature = "gpu")]
 fn main() -> std::process::ExitCode {
+    use bloomery_gpu_gates::record;
+    record::at_main("h2d_probe", record::H2D_PROBE);
     bloomery_gpu_gates::exit_with("h2d_probe", probe::run())
 }
 
@@ -80,6 +92,8 @@ mod probe {
     const PATH_BYTES: usize = 1 << 30;
     /// Timed copies of a one-shot path unless `--reps` says otherwise.
     const PATH_REPS: usize = 8;
+    /// Bytes of a readback unless `--bytes` says otherwise: 32 MiB.
+    const READBACK_BYTES: usize = 32 << 20;
     /// Bytes of a sustained copy and of a loop's source window: one Qwen3.8
     /// layer-batch's routed experts (a whole number of pages).
     const LAYER_BYTES: usize = 1_600_000_000;
@@ -107,10 +121,11 @@ mod probe {
         Attach,
         PageableLoop,
         StagedLoop,
+        Readback,
     }
 
     impl Arm {
-        const ALL: [(&'static str, Arm); 8] = [
+        const ALL: [(&'static str, Arm); 9] = [
             ("pinned", Arm::Pinned),
             ("pageable", Arm::Pageable),
             ("registered", Arm::Registered),
@@ -119,6 +134,7 @@ mod probe {
             ("attach", Arm::Attach),
             ("pageable-loop", Arm::PageableLoop),
             ("staged-loop", Arm::StagedLoop),
+            ("readback", Arm::Readback),
         ];
 
         fn parse(name: &str) -> Result<Arm, GateError> {
@@ -147,6 +163,7 @@ mod probe {
         fn default_bytes(self) -> usize {
             match self {
                 Arm::Pinned | Arm::Pageable | Arm::Registered => PATH_BYTES,
+                Arm::Readback => READBACK_BYTES,
                 _ => LAYER_BYTES,
             }
         }
@@ -552,6 +569,41 @@ mod probe {
         Ok(())
     }
 
+    /// The `readback` arm: each destination's calls, the first and the median.
+    fn readback(stream: &CudaStream, card: &Card, a: &Args) -> Result<(), GateError> {
+        use bloomery_gpu_gates::record::{H2D_READBACK, Record};
+
+        let bytes = a.bytes.unwrap_or(Arm::Readback.default_bytes());
+        let reps = a.reps.unwrap_or(Arm::Readback.default_reps());
+        let dev = DeviceBuffer::<f32>::zeroed(stream, bytes / size_of::<f32>())?;
+        stream.synchronize()?;
+        for dest in ["fresh", "filled"] {
+            let mut ms = Vec::with_capacity(reps);
+            for _ in 0..reps {
+                let t = Instant::now();
+                let host = if dest == "fresh" {
+                    dev.to_host_vec(stream)?
+                } else {
+                    bloomery_gpu::to_host(stream, &dev)?
+                };
+                ms.push(t.elapsed().as_secs_f64() * 1e3);
+                drop(host);
+            }
+            let first = ms[0];
+            ms.sort_by(f64::total_cmp);
+            Record::new(&H2D_READBACK)
+                .w("card", format_args!("{:?}", card.name))
+                .w("bdf", &card.bdf)
+                .w("dest", dest)
+                .u("bytes", bytes)
+                .u("calls", reps)
+                .f("first_ms", first)
+                .f("median_ms", ms[reps / 2])
+                .print();
+        }
+        Ok(())
+    }
+
     fn registered(
         ctx: &Arc<CudaContext>,
         stream: &CudaStream,
@@ -977,6 +1029,8 @@ mod probe {
                     card.print()?;
                     if arm.is_loop() {
                         copy_loop(&ctx, &first, &card, arm, &a)?;
+                    } else if arm == Arm::Readback {
+                        readback(&first, &card, &a)?;
                     } else {
                         one_shot(&ctx, &first, &card, arm, &a)?;
                     }

crates/gpu-gates/src/bin/probe_nvread.rs

@@ -54,17 +54,17 @@
 
 use std::collections::HashMap;
 use std::ops::Range;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::sync::mpsc::{Receiver, Sender, channel};
 use std::thread::JoinHandle;
 use std::time::Instant;
 
 use bloomery_gpu_gates::record::{self, NVREAD, PROBE_NVREAD, Record};
 use bloomery_gpu_gates::{GateError, exit_with, ref_model_path};
-use engram::RowTable;
 use engram::direct::{DIRECT_ALIGN, DirectFile, aligned_span};
-use engram::rows::{PageAdvice, SplitTable};
+use engram::rows::{PageAdvice, PageCount, SplitTable};
+use engram::{EngramError, RowTable};
 use gguf::Split;
 
 /// The `general.architecture` the probe reads.
@@ -420,6 +420,56 @@ struct Part {
     id: u32,
 }
 
+/// A stack's eviction as [`drop_parts`] takes it: [`SplitTable`]'s.
+trait Evict {
+    fn path(&self) -> &Path;
+    fn evict_rows(&self, ids: &[u32]) -> Result<(), EngramError>;
+    fn resident_pages(&self, ids: &[u32]) -> Result<PageCount, EngramError>;
+    fn evict_shard(&self) -> Result<(), EngramError>;
+}
+
+impl Evict for SplitTable {
+    fn path(&self) -> &Path {
+        SplitTable::path(self)
+    }
+
+    fn evict_rows(&self, ids: &[u32]) -> Result<(), EngramError> {
+        SplitTable::evict_rows(self, ids)
+    }
+
+    fn resident_pages(&self, ids: &[u32]) -> Result<PageCount, EngramError> {
+        SplitTable::resident_pages(self, ids)
+    }
+
+    fn evict_shard(&self) -> Result<(), EngramError> {
+        SplitTable::evict_shard(self)
+    }
+}
+
+/// Drop the parts' pages from the process and the page cache: each part's
+/// rows, then the whole shard of a part that kept a page (an edge folio can
+/// cross a row or tensor boundary).
+fn drop_parts(tables: &[impl Evict], parts: &[Part]) -> Result<(), GateError> {
+    for p in parts {
+        tables[p.t].evict_rows(&[p.id])?;
+    }
+    for p in parts {
+        let table = &tables[p.t];
+        let pages = table.resident_pages(&[p.id])?;
+        if pages.resident != 0 {
+            eprintln!(
+                "probe_nvread: retrying eviction over whole shard {}: expert {} kept {} of {} pages",
+                table.path().display(),
+                p.id,
+                pages.resident,
+                pages.total
+            );
+            table.evict_shard()?;
+        }
+    }
+    Ok(())
+}
+
 /// A case's batches.
 #[derive(Clone, Copy)]
 enum Shape {
@@ -542,28 +592,6 @@ impl Rig {
         })
     }
 
-    /// Drop the parts' pages from the process and the page cache.
-    fn drop_parts(&self, parts: &[Part]) -> Result<(), GateError> {
-        for p in parts {
-            self.tables[p.t].evict_rows(&[p.id])?;
-        }
-        for p in parts {
-            let table = &self.tables[p.t];
-            let pages = table.resident_pages(&[p.id])?;
-            if pages.resident != 0 {
-                eprintln!(
-                    "probe_nvread: retrying eviction over whole shard {}: expert {} kept {} of {} pages",
-                    table.path().display(),
-                    p.id,
-                    pages.resident,
-                    pages.total
-                );
-                table.evict_shard()?;
-            }
-        }
-        Ok(())
-    }
-
     /// (resident pages, spanned pages) over the parts.
     fn pages(&self, parts: &[Part]) -> Result<(u64, u64), GateError> {
         let (mut resident, mut total) = (0, 0);
@@ -587,7 +615,7 @@ impl Rig {
             t: self.layers / 2 * STACKS.len(),
             id: id as u32,
         }];
-        self.drop_parts(&part)?;
+        drop_parts(&self.tables, &part)?;
         let (resident, total) = self.pages(&part)?;
         if resident != 0 {
             return Err(format!(
@@ -607,7 +635,7 @@ impl Rig {
             )
             .into());
         }
-        self.drop_parts(&part)
+        drop_parts(&self.tables, &part)
     }
 
     /// The drawn batch of `shape`, its picks as labels, its parts dropped and
@@ -652,7 +680,7 @@ impl Rig {
                     (parts, vec![layer.to_string()])
                 }
             };
-            self.drop_parts(&parts)?;
+            drop_parts(&self.tables, &parts)?;
             let (resident, _) = self.pages(&parts)?;
             if resident == 0 {
                 return Ok((parts, labels));
@@ -779,7 +807,7 @@ impl Rig {
             .into());
         }
         s.asked[0] = total * self.tables[parts[0].t].page_bytes();
-        self.drop_parts(parts)?;
+        drop_parts(&self.tables, parts)?;
         let reads = self.reads(parts)?;
         s.asked[1] = reads.iter().map(|r| r.span as u64).sum();
         s.asked[2] = s.asked[1];
@@ -863,8 +891,45 @@ impl Rig {
 
 #[cfg(test)]
 mod tests {
+    use std::cell::Cell;
+
     use super::*;
 
+    /// A stack whose row eviction keeps one page of the row, as an edge folio
+    /// that crosses into a neighbour does; only its whole shard's eviction
+    /// drops it.
+    struct EdgeFolio(Cell<u64>);
+
+    impl Evict for EdgeFolio {
+        fn path(&self) -> &Path {
+            Path::new("edge.gguf")
+        }
+
+        fn evict_rows(&self, _: &[u32]) -> Result<(), EngramError> {
+            self.0.set(self.0.get().min(1));
+            Ok(())
+        }
+
+        fn resident_pages(&self, _: &[u32]) -> Result<PageCount, EngramError> {
+            Ok(PageCount {
+                resident: self.0.get(),
+                total: 4,
+            })
+        }
+
+        fn evict_shard(&self) -> Result<(), EngramError> {
+            self.0.set(0);
+            Ok(())
+        }
+    }
+
+    #[test]
+    fn a_retained_edge_folio_leaves_with_its_shard() {
+        let tables = [EdgeFolio(Cell::new(4))];
+        drop_parts(&tables, &[Part { t: 0, id: 3 }]).unwrap();
+        assert_eq!(tables[0].resident_pages(&[3]).unwrap().resident, 0);
+    }
+
     #[test]
     fn partition_covers_every_item_once_in_even_ranges() {
         for (len, n) in [(9, 8), (3, 8), (768, 8), (5, 1), (0, 4), (8, 8)] {