diff --git a/crates/gpu-deepseek41/src/chain/ffn/batch.rs b/crates/gpu-deepseek41/src/chain/ffn/batch.rs index c5579f7e2..326f108c3 100644 --- a/crates/gpu-deepseek41/src/chain/ffn/batch.rs +++ b/crates/gpu-deepseek41/src/chain/ffn/batch.rs @@ -2769,3 +2769,218 @@ fn count_of(by_count: &mut [T], m: usize) -> Result<&mut T, GpuError> { detail: format!("a chunk of {m} tokens; the batch holds scratch for 1..={n}"), }) } + +#[cfg(test)] +mod tests { + use super::{CardAccTier, FfnBatchKernels}; + use bloomery_gpu::host::handoff::{Handoff, HandoffKernels}; + use bloomery_gpu::host::page::HandoffLayout; + use bloomery_gpu::host::step::HandoffTarget; + use bloomery_gpu::host::tier::{TierImageLayout, TierTarget}; + use bloomery_gpu::hybrid::HOST; + use bloomery_gpu::{FaultSite, Gpu, GpuError, Window}; + use cuda_core::DeviceBuffer; + + /// The handoff's places feed the production tier sum, not a readback + /// alone: every routed row belongs to exactly one of host, card and + /// tier, and all other copies are poisoned. Both slot entries must + /// conserve that union, including the final routed lane and partial + /// blocks, eagerly and on replay. Invalid ids must still raise. + #[test] + #[ignore = "needs a CUDA device; `just gate-gpu-ds41-lib` runs it on the box"] + fn hw_f32_tier_handoffs_conserve_consumed_routes() { + const EXPERTS: usize = 16; + const ROW_OFF: usize = 3; + const LAYER: usize = 7; + let gpu = Gpu::new().expect("a GPU"); + let stream = gpu.stream(); + let handoff = HandoffKernels::load(gpu.context()).expect("handoff kernels"); + let sums = FfnBatchKernels::load(gpu.context()).expect("tier sum kernels"); + let mut map = vec![HOST - 1; ROW_OFF + EXPERTS]; + let mut tmap = map.clone(); + for id in 0..EXPERTS { + map[ROW_OFF + id] = if id % 3 == 0 { (id / 3) as u32 } else { HOST }; + tmap[ROW_OFF + id] = if id % 3 == 1 { (id / 3) as u32 } else { HOST }; + } + let map = DeviceBuffer::from_host(stream, &map).expect("card map"); + let tmap = DeviceBuffer::from_host(stream, &tmap).expect("tier map"); + for n_used in [4, 6, 8] { + for hidden in [n_used, 256, 257] { + for bad in [None, Some(EXPERTS as u32), Some(u32::MAX)] { + let mut ids = vec![15, 0, 10, 5, 8, 1, 2, 7]; + ids.truncate(n_used); + if let Some(id) = bad { + ids[n_used - 1] = id; + } + // No handoff may read the router's trailing lane. + ids.push(u32::MAX); + let weights: Vec = (0..n_used) + .map(|s| (s + 1) as f32 / 16.0) + .chain([f32::NAN]) + .collect(); + let x: Vec = (0..hidden).map(|d| (d % 7 + 1) as f32 / 8.0).collect(); + let row = |s: usize, d: usize| (ids[s] + 1) as f32 * x[d]; + let mut down = vec![f32::NAN; n_used * hidden]; + let mut trows = down.clone(); + let mut host = vec![0.0f32; hidden]; + let mut want = host.clone(); + for d in 0..hidden { + let mut card_tier = 0.0f32; + for s in 0..n_used { + if ids[s] >= EXPERTS as u32 { + continue; + } + let value = row(s, d); + match ids[s] % 3 { + 0 => down[s * hidden + d] = value, + 1 => trows[s * hidden + d] = value, + _ => host[d] = value.mul_add(weights[s], host[d]), + } + if ids[s] % 3 != 2 { + card_tier = value.mul_add(weights[s], card_tier); + } + } + want[d] = card_tier + host[d]; + } + let ids_d = DeviceBuffer::from_host(stream, &ids).expect("ids"); + let mut weights_d = DeviceBuffer::from_host(stream, &weights).expect("weights"); + let x_d = DeviceBuffer::from_host(stream, &x).expect("activation"); + let seq = DeviceBuffer::from_host(stream, &[19u32]).expect("sequence"); + let down = DeviceBuffer::from_host(stream, &down).expect("card rows"); + let trows = DeviceBuffer::from_host(stream, &trows).expect("tier rows"); + let doubled_host: Vec = host.iter().map(|h| 2.0 * h).collect(); + let mut host = DeviceBuffer::from_host(stream, &host).expect("host sum"); + let lay = HandoffLayout { + seq: 0, + ids: 1, + weights: 1 + n_used, + x: 1 + 2 * n_used, + n_used, + hidden, + }; + let tl = TierImageLayout { + sel: 0, + q3: 0, + d8: 0, + x: n_used, + n_used, + q3_words: 0, + d8_len: 0, + x_len: hidden, + hidden, + cols: 1, + }; + for captured in [false, true] { + let mut image = + DeviceBuffer::zeroed(stream, lay.x + hidden).expect("image"); + let mut timage = + DeviceBuffer::zeroed(stream, tl.x + hidden).expect("tier image"); + let poison = vec![HOST - 1; n_used + 1]; + let mut sel = DeviceBuffer::from_host(stream, &poison).expect("places"); + let mut tsel = + DeviceBuffer::from_host(stream, &poison).expect("tier places"); + let mut acc = DeviceBuffer::zeroed(stream, hidden).expect("sum"); + let mut result = DeviceBuffer::zeroed(stream, hidden).expect("result"); + let mut enqueue = || -> Result<(), GpuError> { + handoff.enqueue_handoff_tier( + stream, + &Handoff { + ids: &ids_d, + weights: &weights_d, + map: &map, + row_off: ROW_OFF, + n_expert: EXPERTS, + }, + &tmap, + ( + HandoffTarget { + x: &x_d, + seq: &seq, + image: &mut image, + layout: lay, + }, + TierTarget { + image: &mut timage, + rows: &trows, + layout: tl, + }, + ), + gpu.layer_sink(LAYER)?, + (&mut sel, &mut tsel), + )?; + let routed_weights = + Window::::of(&image, 4 * lay.weights, n_used)?; + sums.enqueue_card_acc_tier( + stream, + &CardAccTier { + down: &down, + trows: &trows, + w: &routed_weights, + sel: &sel, + tsel: &tsel, + n: hidden, + m: 1, + n_card: EXPERTS.div_ceil(3), + n_tier: EXPERTS.div_ceil(3), + n_used, + }, + &mut acc, + )?; + gpu.elem() + .enqueue_add(stream, &acc, &host, hidden, &mut result) + }; + if captured { + let graph = gpu.capture(|_| enqueue()).expect("capture"); + graph.launch(stream).expect("replay"); + let doubled_weights: Vec = + weights.iter().map(|w| 2.0 * w).collect(); + weights_d + .copy_from_host(stream, &doubled_weights) + .expect("new route weights"); + host.copy_from_host(stream, &doubled_host) + .expect("new host sum"); + graph.launch(stream).expect("second replay"); + stream.synchronize().expect("replays settled"); + } else { + enqueue().expect("eager handoff and consumption"); + } + let scale = if captured { 2.0 } else { 1.0 }; + let got = result.to_host_vec(stream).expect("result"); + let places = sel.to_host_vec(stream).expect("places"); + let tier_places = tsel.to_host_vec(stream).expect("tier places"); + let image = image.to_host_vec(stream).expect("host image"); + let timage = timage.to_host_vec(stream).expect("tier image"); + stream.synchronize().expect("settled"); + let fault = gpu.take_fault().expect("fault word"); + if bad.is_some() { + let fault = fault.expect("an invalid id must raise"); + assert_eq!(fault.layer, LAYER as u32); + assert_eq!(fault.code, FaultSite::ExpertId as u32); + assert_eq!(fault.sites, 1 << FaultSite::ExpertId as u32); + } else { + assert!(fault.is_none(), "{fault:?}"); + } + assert_eq!(places[n_used], HOST - 1, "card places tail"); + assert_eq!(tier_places[n_used], HOST - 1, "tier places tail"); + assert_eq!(image[lay.seq], 19, "sequence"); + assert_eq!(&image[lay.ids..lay.ids + n_used], &ids[..n_used]); + for s in 0..n_used { + assert_eq!(image[lay.weights + s], (weights[s] * scale).to_bits()); + assert_eq!(timage[tl.sel + s], tier_places[s]); + } + for d in 0..hidden { + assert_eq!(image[lay.x + d], x[d].to_bits()); + assert_eq!(timage[tl.x + d], x[d].to_bits()); + } + assert!( + got.iter() + .zip(&want) + .all(|(g, w)| g.to_bits() == (w * scale).to_bits()), + "{n_used} slots, width {hidden}, bad {bad:?}, capture {captured}: result conservation" + ); + } + } + } + } + } +} diff --git a/crates/gpu/src/host/handoff.rs b/crates/gpu/src/host/handoff.rs index 1e36a2d0a..780914af6 100644 --- a/crates/gpu/src/host/handoff.rs +++ b/crates/gpu/src/host/handoff.rs @@ -18,10 +18,10 @@ //! one launch ([`HandoffKernels::enqueue_handoff_cols`]); //! [`HandoffKernels::enqueue_places_cols`] writes the places alone of such a //! row, for a walk whose routing reaches the host by a download, through -//! `crate::card_slots`' `ds41_ffn_places`. `ds41_ffn_handoff_8_tier` and -//! `ds41_ffn_handoff_4_tier` are the eight- and four-slot handoffs of a layer -//! an expert tier holds experts of: the -//! same writes, and the slots' tier places and the activation in f32 into +//! `crate::card_slots`' `ds41_ffn_places`. `ds41_ffn_handoff_4_tier`, +//! `ffn_handoff_6_tier` and `ds41_ffn_handoff_8_tier` hand over four, six or eight +//! slots of a layer an expert tier holds experts of: the same writes, the slots' +//! tier places and the activation in f32 into //! the row's tier image ([`HandoffKernels::enqueue_handoff_tier`]); //! `ds41_ffn_handoff_10_cols_tier` is `_10_cols`'s of such a layer, every //! column's tier places and activation into the row's tier image @@ -282,6 +282,88 @@ mod handoff_kernels { unsafe { handoff_at::(&a, d, &mut image, &mut sel) }; } + /// [`ds41_ffn_handoff`] and the row's f32 tier image, with the same + /// per-thread writes and following go as [`ds41_ffn_handoff_8_tier`]. + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(256)] + #[launch_contract( + domain = 1, + block = (256, 1, 1), + requires = ( + ids_in.len() >= 6, + w_in.len() >= 6, + map.len() >= row_off + n_expert, + tmap.len() >= row_off + n_expert, + x.len() >= n, + seq.len() >= 1, + n >= 6, + ids_at >= seq_at + 1, + wts_at >= ids_at + 6, + x_at >= wts_at + 6, + image.len() >= x_at + n, + tx_at >= tsel_at + 6, + timage.len() >= tx_at + n, + sel.len() >= 6, + tsel.len() >= 6 + ) + )] + pub fn ffn_handoff_6_tier( + ids_in: &[u32], + w_in: &[f32], + map: &[u32], + tmap: &[u32], + row_off: u32, + n_expert: u32, + x: &[f32], + seq: &[u32], + n: u32, + seq_at: u32, + ids_at: u32, + wts_at: u32, + x_at: u32, + tsel_at: u32, + tx_at: u32, + fault: FaultSink, + mut image: DisjointSlice, + mut timage: DisjointSlice, + mut sel: DisjointSlice, + mut tsel: DisjointSlice, + ) { + let d = thread::index_1d().get(); + if d >= n as usize { + return; + } + let a = HandoffIn { + ids_in, + w_in, + map, + row_off, + n_expert, + x, + seq, + seq_at, + ids_at, + wts_at, + x_at, + fault, + }; + // SAFETY: d < n, and the launch contract is `handoff_at`'s at + // N_USED slots; thread d is the launch's only thread at d. + unsafe { handoff_at::(&a, d, &mut image, &mut sel) }; + let t = TierIn { + tmap, + tsel_at, + tx_at, + }; + // SAFETY: d < n, and the launch contract is `handoff_tier_at`'s at + // N_USED slots; thread d is the launch's only thread at d. + unsafe { handoff_tier_at::(&a, &t, d, &mut timage, &mut tsel) }; + } + /// [`ds41_ffn_handoff_8`] and the row's f32 tier image /// ([`handoff_tier_at`]): beside the handoff and the stage card's places, /// threads `s < 8` write `tsel[s] = tmap[row_off + id]` — the id's slot @@ -1265,8 +1347,9 @@ impl HandoffKernels { } impl HandoffKernels { - /// Enqueue the handoff of a tier layer of four or eight slots a token - /// (`ds41_ffn_handoff_4_tier`, `ds41_ffn_handoff_8_tier`): [`HandoffKernels::enqueue_handoff`]'s + /// Enqueue the handoff of a tier layer of four, six or eight slots a token + /// (`ds41_ffn_handoff_4_tier`, `ffn_handoff_6_tier` or `ds41_ffn_handoff_8_tier`): + /// [`HandoffKernels::enqueue_handoff`]'s /// writes into `target`, and each slot's tier place (from `tmap`, the /// slot map's tier copy at the card copy's row offsets) into `tsel` and /// with the activation into the f32 tier image `ttarget`. A layout of @@ -1286,79 +1369,111 @@ impl HandoffKernels { fault: FaultSink, (sel, tsel): (&mut DeviceBuffer, &mut DeviceBuffer), ) -> Result<(), GpuError> { - const WHAT: &str = "ds41_ffn_handoff_tier"; let (lay, tl) = (target.layout, ttarget.layout); - if !matches!(lay.n_used, SLOTS_4 | SLOTS_8) - || tl.n_used != lay.n_used - || tl.x_len != lay.hidden - || tl.q3_words != 0 - || tl.d8_len != 0 - { - return Err(GpuError::Shape { - what: WHAT, - detail: format!( - "the boundary carries {} slots of {} values and the tier image {} slots, {} \ - f32, {} code words, {} scales; the entries hand over {SLOTS_4} or {SLOTS_8} \ - slots, the tier image the boundary's, and the activation in f32", - lay.n_used, lay.hidden, tl.n_used, tl.x_len, tl.q3_words, tl.d8_len - ), - }); - } + let slots = check_tier(&lay, &tl)?; + let what = "F32 tier handoff"; let n = lay.hidden; - let grid = launch_u32(WHAT, "grid", n.div_ceil(HANDOFF_THREADS as usize))?; + let grid = launch_u32(what, "grid", n.div_ceil(HANDOFF_THREADS as usize))?; let cfg = LaunchConfig1D::new(grid, HANDOFF_THREADS, 0); - if lay.n_used == SLOTS_4 { - let prep = self.module.prepare_ds41_ffn_handoff_4_tier(cfg)?; - self.module.ds41_ffn_handoff_4_tier( - stream, - &prep, - h.ids, - h.weights, - h.map, - tmap, - launch_u32(WHAT, "row_off", h.row_off)?, - launch_u32(WHAT, "n_expert", h.n_expert)?, - target.x, - target.seq, - launch_u32(WHAT, "n", n)?, - launch_u32(WHAT, "seq_at", lay.seq)?, - launch_u32(WHAT, "ids_at", lay.ids)?, - launch_u32(WHAT, "wts_at", lay.weights)?, - launch_u32(WHAT, "x_at", lay.x)?, - launch_u32(WHAT, "tsel_at", tl.sel)?, - launch_u32(WHAT, "tx_at", tl.x)?, - fault, - target.image, - ttarget.image, - sel, - tsel, - )?; - } else { - let prep = self.module.prepare_ds41_ffn_handoff_8_tier(cfg)?; - self.module.ds41_ffn_handoff_8_tier( - stream, - &prep, - h.ids, - h.weights, - h.map, - tmap, - launch_u32(WHAT, "row_off", h.row_off)?, - launch_u32(WHAT, "n_expert", h.n_expert)?, - target.x, - target.seq, - launch_u32(WHAT, "n", n)?, - launch_u32(WHAT, "seq_at", lay.seq)?, - launch_u32(WHAT, "ids_at", lay.ids)?, - launch_u32(WHAT, "wts_at", lay.weights)?, - launch_u32(WHAT, "x_at", lay.x)?, - launch_u32(WHAT, "tsel_at", tl.sel)?, - launch_u32(WHAT, "tx_at", tl.x)?, - fault, - target.image, - ttarget.image, - sel, - tsel, - )?; + let row_off = launch_u32(what, "row_off", h.row_off)?; + let n_expert = launch_u32(what, "n_expert", h.n_expert)?; + let at = [ + launch_u32(what, "n", n)?, + launch_u32(what, "seq_at", lay.seq)?, + launch_u32(what, "ids_at", lay.ids)?, + launch_u32(what, "wts_at", lay.weights)?, + launch_u32(what, "x_at", lay.x)?, + launch_u32(what, "tsel_at", tl.sel)?, + launch_u32(what, "tx_at", tl.x)?, + ]; + match slots { + HandoffSlots::Four => { + let prep = self.module.prepare_ds41_ffn_handoff_4_tier(cfg)?; + self.module.ds41_ffn_handoff_4_tier( + stream, + &prep, + h.ids, + h.weights, + h.map, + tmap, + row_off, + n_expert, + target.x, + target.seq, + at[0], + at[1], + at[2], + at[3], + at[4], + at[5], + at[6], + fault, + target.image, + ttarget.image, + sel, + tsel, + )?; + } + HandoffSlots::Six => { + let prep = self.module.prepare_ffn_handoff_6_tier(cfg)?; + self.module.ffn_handoff_6_tier( + stream, + &prep, + h.ids, + h.weights, + h.map, + tmap, + row_off, + n_expert, + target.x, + target.seq, + at[0], + at[1], + at[2], + at[3], + at[4], + at[5], + at[6], + fault, + target.image, + ttarget.image, + sel, + tsel, + )?; + } + HandoffSlots::Eight => { + let prep = self.module.prepare_ds41_ffn_handoff_8_tier(cfg)?; + self.module.ds41_ffn_handoff_8_tier( + stream, + &prep, + h.ids, + h.weights, + h.map, + tmap, + row_off, + n_expert, + target.x, + target.seq, + at[0], + at[1], + at[2], + at[3], + at[4], + at[5], + at[6], + fault, + target.image, + ttarget.image, + sel, + tsel, + )?; + } + HandoffSlots::Nine | HandoffSlots::Ten => { + return Err(GpuError::shape( + "HandoffKernels::enqueue_handoff_tier", + format!("no F32 tier handoff entry for {slots:?}"), + )); + } } Ok(()) } @@ -1514,6 +1629,31 @@ impl HandoffKernels { } } +/// The single-token F32 tier handoff's layout contract, shared by its +/// slot entries. The generated launch checks the input and output spans. +fn check_tier(lay: &HandoffLayout, tl: &TierImageLayout) -> Result { + let slots = HandoffSlots::of(lay.n_used); + if !matches!( + slots, + Ok(HandoffSlots::Four | HandoffSlots::Six | HandoffSlots::Eight) + ) || tl.n_used != lay.n_used + || tl.x_len != lay.hidden + || tl.q3_words != 0 + || tl.d8_len != 0 + { + return Err(GpuError::Shape { + what: "HandoffKernels::enqueue_handoff_tier", + detail: format!( + "the boundary carries {} slots of {} values and the tier image {} slots, {} \ + f32, {} code words, {} scales; the entries hand over matching 4, 6 or 8 slots \ + and the activation in f32", + lay.n_used, lay.hidden, tl.n_used, tl.x_len, tl.q3_words, tl.d8_len + ), + }); + } + slots.map_err(|e| GpuError::shape("HandoffKernels::enqueue_handoff_tier", e.to_string())) +} + /// [`HandoffKernels::enqueue_handoff_cols`]'s refusals, by name before the /// launch: a layout of another slot count, `m` of 0, a pitch under ten, and a /// `sel` of `sel_len` places shorter than `m` columns of ten. @@ -1600,7 +1740,7 @@ pub struct Places<'a> { #[cfg(test)] mod tests { - use super::{HandoffRefused, HandoffSlots, check_cols, check_cols_tier}; + use super::{HandoffRefused, HandoffSlots, check_cols, check_cols_tier, check_tier}; use crate::host::page::HandoffLayout; use crate::host::tier::TierImageLayout; @@ -1651,6 +1791,44 @@ mod tests { } } + #[test] + fn a_token_tier_handoff_requires_matching_f32_slot_images() { + for n_used in 0..=10 { + let lay = HandoffLayout { + n_used, + ..boundary() + }; + for tier_slots in 0..=10 { + let tl = TierImageLayout { + n_used: tier_slots, + x: tier_slots, + ..image(1) + }; + assert_eq!( + check_tier(&lay, &tl).is_ok(), + matches!(n_used, 4 | 6 | 8) && tier_slots == n_used, + "host {n_used}, tier {tier_slots}" + ); + } + } + for n_used in [4, 6, 8] { + let lay = HandoffLayout { + n_used, + ..boundary() + }; + let tl = TierImageLayout { + n_used, + x: n_used, + ..image(1) + }; + for x_len in [0, HIDDEN - 1, HIDDEN + 1, 2 * HIDDEN] { + assert!(check_tier(&lay, &TierImageLayout { x_len, ..tl }).is_err()); + } + assert!(check_tier(&lay, &TierImageLayout { q3_words: 1, ..tl }).is_err()); + assert!(check_tier(&lay, &TierImageLayout { d8_len: 1, ..tl }).is_err()); + } + } + /// A tier handoff of `m` columns over a page of 8 refuses, by name, a /// `tsel` or a `sel` shorter than `10·m`, and takes ones of exactly that: /// the launch writes every column's places. diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 0d60c5feb..72eb230ce 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -2,8 +2,31 @@ Design snapshot: **`e53cabebbb9b16249c81b7c26a026da1670f7d9d`**, requested by [#632](https://github.com/midagedev/bloomery/issues/632). The source-gap observations below describe -that snapshot. The current tree additionally has a **header-only reader**; it still has no Flint -body, app session or serve seat. No GPU, inference, performance or weight-content validation is claimed. +that snapshot. This component tree additionally has a header-only reader and shared tier-handoff +work, but no Flint body, app session or serve seat. Primitive GPU evidence is not Flint model admission. + +## Six-slot handoff closure checks — 2026-10-26 + +The existing #781 branch was ordinarily merged with main +`15778e2cb2dd7af428d2dd14259e6b0ea8ca6b89`; its published history was not rewritten. +On source `9bfa55adc257f7e6aae4952aca06246eddaeef70`, the owning DS41 library ran on +Hikari's RTX 3090 (GPU0): **38 passed, 0 failed, 0 ignored**, in 28.75 seconds. +`hw_f32_tier_handoffs_conserve_consumed_routes` exercised four, six and eight routes, +including partial widths, invalid IDs, poisoned unconsumed rows and changed-input graph replay. +The following host-only changes format the guard and replace a redundant slot-dependent +error label with the common `F32 tier handoff` context. The measured common-name census +falls from 39 to 37; the ABI-stable generated kernel names and dispatch stay unchanged. +The earlier GPU run is not relabeled as a run of these later commits. + +The actual `generate_ds41` carrier at `b9b6e013ebfab5748688be52e9a5d398ac2fc9a3` +subsequently measured `ffn_handoff_6_tier` on sm_86 / physical cc8.6: +ptxas13.3.33 spill0, driver-JIT local0, both register counts26 (driver595.91.07, +JIT CUDA13.2, Rust1.99.1 / cuda-oxide9df9e7ce / repository patch31ab6a1d3f58). +Only that measured row is added. Its normalized entry digest is +`a176a9a5846778e40ba99fc2be637d90` (decl1, 163 lines). +Measurements establish only physical cc8.6. The other three +sm_86/sm_121/sm_121a + physical cc12.1 rows remain unmeasured: they carry the entry at the cc8.6 +values under a `PIN(2026-10-27)` until a GB10 scan runs. ## Host reader status @@ -66,6 +89,33 @@ Each device slice needs its numerical/fault gates and the maintainer's offered h Legacy metadata booleans share `RouterNorm::from_legacy_bool`: false keeps scores and true uses the existing epsilon-added selected sum. Flint's distinct floor mapping stays in its reader. +### Six-slot F32 tier handoff implementation + +`HandoffKernels::enqueue_handoff_tier` now selects a genuine six-slot F32 entry +beside the existing four- and eight-slot entries. Both host and tier layouts must have the +same supported slot count and activation width; compressed tier images remain +refused. The entry uses the existing copy/place/fault helpers and the existing +subsequent go/publication protocol; no Flint router or Program is registered. + +`hw_f32_tier_handoffs_conserve_consumed_routes`, owned by +`gate-gpu-ds41-lib`, consumes the four-, six- and eight-slot handoffs' weights and emitted +places through the production tier accumulator and adds the host sum. It covers partial +widths, invalid ids, poisoned non-owning rows, trailing sentinels and changed replay +weights. Its host sum is precomputed and activation images are checked by +readback: it is not expert execution or two-card completion/publication proof. +Earlier integrated-source results do not verify this branch after its merge with current +main; the final-source hardware result must be recorded separately. + +The new `ffn_handoff_6_tier` entry needs add-class PTX/JIT/resource and +owning-gate evidence; the four-slot, eight-slot and other existing entries must remain +unchanged. The cc12.1 tables pin it at the cc8.6 scan's values until an observed NVIDIA JIT run. +Actual six-slot body/tier and two-card completion remain integration work, not +capabilities inferred from this primitive. +`check-ptx-names` is green with those pinned rows; a GB10 scan replaces them with the +observed spill/JIT-local measurements. Offline no-JIT scans cannot supply that column. + + + ## 1. Sources and observed artifact diff --git a/tools/model-tax.tsv b/tools/model-tax.tsv index 148968879..fe9af6618 100644 --- a/tools/model-tax.tsv +++ b/tools/model-tax.tsv @@ -131,7 +131,7 @@ crates/gpu-vision/src/encoder.rs deepseek 2 baseline D 2: media design: VideoRul crates/gpu/src/arch/hybrid.rs nemotron 1 baseline O 1: own path: an alias row in tools/model-tokens.tsv or a move into the family directory crates/gpu/src/card_slots.rs deepseek 24 PIN(2026-10-21) deepseek: baseline K 20 + 4, the four-slot entry `ds41_ffn_card_acc_4` (MiniMax-M3 routes 4 a token): one more copy of the per-count entry, since a `#[kernel]` entry cannot be generic over the slot count; the deferred rename of the entries by what they compute absorbs it crates/gpu/src/dspark/head.rs deepseek 3 baseline K 3: rename the kernel entries by what they compute (deferred) -crates/gpu/src/host/handoff.rs deepseek 38 PIN(2026-10-24) deepseek: baseline K 31 + 4 + 3, the four-slot tier entry `ds41_ffn_handoff_4_tier` (MiniMax-M3.1's bp) beside the four-slot entry `ds41_ffn_handoff_4` (MiniMax-M3 routes 4 a token): one more copy of the per-count entry, since a `#[kernel]` entry cannot be generic over the slot count; the deferred rename of the entries by what they compute absorbs it +crates/gpu/src/host/handoff.rs deepseek 37 PIN(2026-10-26) deepseek: the tier launcher uses a common F32 error context instead of selecting a family-named label per slot count; existing kernel entry symbols remain ABI-stable crates/hf/src/vocab.rs qwen 4 PIN(2026-10-20) qwen: the qwen35 tokenizer pipeline name and regex; tokenizer programs are not yet described by ModelSpec crates/metal/src/bin/bloomery_serve_metal.rs qwen 3 PIN(2026-10-23) qwen: Metal arch dispatch by program: the Metal bins open the one Metal architecture's session by name until crates/metal/src/arch picks a session by the file's architecture string crates/metal/src/bin/generate_metal.rs qwen 2 PIN(2026-10-23) qwen: Metal arch dispatch by program: the Metal bins open the one Metal architecture's session by name until crates/metal/src/arch picks a session by the file's architecture string diff --git a/tools/ref/ptx-shapes.tsv b/tools/ref/ptx-shapes.tsv index 07613d074..b038d2937 100644 --- a/tools/ref/ptx-shapes.tsv +++ b/tools/ref/ptx-shapes.tsv @@ -74,6 +74,10 @@ generate_ds41 ds41_ffn_handoff_10_cols_tier 0 0 generate_ds41 ds41_ffn_handoff_4 0 0 generate_ds41 ds41_ffn_handoff_8 0 0 generate_ds41 ds41_ffn_handoff_4_tier 0 0 +# PIN(2026-10-26): ffn_handoff_6_tier measured from generate_ds41 at b9b6e013 with +# Rust1.99.1 / cuda-oxide9df9e7ce + repository patch31ab6a1d3f58; sm_86 ptxas13.3.33, +# RTX3090 cc8.6 driver595.91.07 JIT CUDA13.2: spill0 / jit_local0 (both register counts26). +generate_ds41 ffn_handoff_6_tier 0 0 generate_ds41 ds41_ffn_handoff_8_tier 0 0 generate_ds41 ds41_ffn_handoff_9 0 0 generate_ds41 ds41_ffn_handoff_tier 0 0 diff --git a/tools/ref/ptx-shapes/sm_121-cc12.1.tsv b/tools/ref/ptx-shapes/sm_121-cc12.1.tsv index b31394183..850f9ecdf 100644 --- a/tools/ref/ptx-shapes/sm_121-cc12.1.tsv +++ b/tools/ref/ptx-shapes/sm_121-cc12.1.tsv @@ -70,6 +70,8 @@ generate_ds41 ds41_ffn_handoff_10_cols_tier 0 0 generate_ds41 ds41_ffn_handoff_4 0 0 generate_ds41 ds41_ffn_handoff_8 0 0 generate_ds41 ds41_ffn_handoff_4_tier 0 0 +# PIN(2026-10-27): ffn_handoff_6_tier at the cc 8.6 scan's value (tools/ref/ptx-shapes.tsv), not measured on cc 12.1, until a GB10 scan runs. +generate_ds41 ffn_handoff_6_tier 0 0 generate_ds41 ds41_ffn_handoff_8_tier 0 0 generate_ds41 ds41_ffn_handoff_tier 0 0 generate_ds41 ds41_ffn_places 0 0 diff --git a/tools/ref/ptx-shapes/sm_121a-cc12.1.tsv b/tools/ref/ptx-shapes/sm_121a-cc12.1.tsv index 500bf10e5..c951ae267 100644 --- a/tools/ref/ptx-shapes/sm_121a-cc12.1.tsv +++ b/tools/ref/ptx-shapes/sm_121a-cc12.1.tsv @@ -85,6 +85,8 @@ generate_ds41 ds41_ffn_handoff_10_cols_tier 0 0 generate_ds41 ds41_ffn_handoff_4 0 0 generate_ds41 ds41_ffn_handoff_8 0 0 generate_ds41 ds41_ffn_handoff_4_tier 0 0 +# PIN(2026-10-27): ffn_handoff_6_tier at the cc 8.6 scan's value (tools/ref/ptx-shapes.tsv), not measured on cc 12.1, until a GB10 scan runs. +generate_ds41 ffn_handoff_6_tier 0 0 generate_ds41 ds41_ffn_handoff_8_tier 0 0 generate_ds41 ds41_ffn_handoff_9 0 0 generate_ds41 ds41_ffn_handoff_tier 0 0 diff --git a/tools/ref/ptx-shapes/sm_86-cc12.1.tsv b/tools/ref/ptx-shapes/sm_86-cc12.1.tsv index bf7802d0f..de7cb39c0 100644 --- a/tools/ref/ptx-shapes/sm_86-cc12.1.tsv +++ b/tools/ref/ptx-shapes/sm_86-cc12.1.tsv @@ -69,6 +69,8 @@ generate_ds41 ds41_ffn_handoff_10_cols_tier 0 0 generate_ds41 ds41_ffn_handoff_4 0 0 generate_ds41 ds41_ffn_handoff_8 0 0 generate_ds41 ds41_ffn_handoff_4_tier 0 0 +# PIN(2026-10-27): ffn_handoff_6_tier at the cc 8.6 scan's value (tools/ref/ptx-shapes.tsv), not measured on cc 12.1, until a GB10 scan runs. +generate_ds41 ffn_handoff_6_tier 0 0 generate_ds41 ds41_ffn_handoff_8_tier 0 0 generate_ds41 ds41_ffn_handoff_tier 0 0 generate_ds41 ds41_ffn_places 0 0 diff --git a/tools/unsafe-ratchet.txt b/tools/unsafe-ratchet.txt index 292bd3c21..932801ead 100644 --- a/tools/unsafe-ratchet.txt +++ b/tools/unsafe-ratchet.txt @@ -623,7 +623,10 @@ gguf 7 2 0 # route tile's column slots and input scales in shared memory (nvfp4mma.rs `TileColumns::put` and `get`, the two # kernels' column allocations, their writes ahead of the K walk and their epilogues' reads), counted by # tools/check-unsafe.sh. Each new block has its SAFETY line. -gpu 2691 346 10 +# PIN(2026-10-26): gpu 2691 -> 2693 blocks, the six-slot F32 tier entry adds two blocks: +# ffn_handoff_6_tier calls handoff_at and handoff_tier_at under its checked launch contract; +# both blocks state their per-thread bounds and disjoint-write SAFETY invariant. +gpu 2693 346 10 # PIN(2026-10-01): gpu-deepseek41 459 -> 464 blocks, 51 -> 52 fns, a new kernel entry, not old code relaxed: # ds41_ffn_card_acc_8_tier (the tier form of the card sum at 8 slots, chain/ffn/batch.rs) adds its two blocks and # card_acc_tier_at (the one unsafe fn) with its three, each with its SAFETY line.