diff --git a/crates/gpu-deepseek41/src/body.rs b/crates/gpu-deepseek41/src/body.rs index 1eb242155..dbae7361f 100644 --- a/crates/gpu-deepseek41/src/body.rs +++ b/crates/gpu-deepseek41/src/body.rs @@ -2583,7 +2583,7 @@ impl Body { // The tiers' batch staging is reserved on their cards by the plan, // so it is made at load, not at the first prompt. hybrid.prepare_batch(gpu.context(), prefill::T_MAX)?; - hybrid.check_tier_reserves(plan.machine, &cards)?; + hybrid.check_tier_reserves(plan, &cards)?; Some(TierPiece::new(gpu, hybrid.slots(), PAIR_ROWS)?) }; let ced = ced::Ced::new(&CedLayer::table(hp), seq.ring_rows(), cfg.ced); diff --git a/crates/gpu-gates/src/bin/gate_glm5next_nodes.rs b/crates/gpu-gates/src/bin/gate_glm5next_nodes.rs index 1b025c8c0..2e593a526 100644 --- a/crates/gpu-gates/src/bin/gate_glm5next_nodes.rs +++ b/crates/gpu-gates/src/bin/gate_glm5next_nodes.rs @@ -30,7 +30,7 @@ mod gate { let file = Split::open(&path)?; let inputs = PlanInputs::describe(&file)?; let (mut machine, node) = - crate::node_history::machine(inputs.model.layers, tier_batch(&inputs))?; + crate::node_history::machine(inputs.model.layers, tier_batch(&inputs)?)?; reserve_checkpoints( &mut machine, Seqs::priced(1, checkpoint_price(&inputs)?), diff --git a/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs b/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs index 4fc7a7a82..3d30bbeda 100644 --- a/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs +++ b/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs @@ -517,7 +517,7 @@ mod gate { fn bp_machine(inputs: &PlanInputs) -> Result Machine + Copy, GateError> { glm_machine( Place::Bp, - Some(place::tier_batch(inputs)), + Some(place::tier_batch(inputs)?), 1, checkpoint_price(inputs)?, u64::try_from(CTX)?, diff --git a/crates/gpu-gates/src/bin/gate_mimo2_serve.rs b/crates/gpu-gates/src/bin/gate_mimo2_serve.rs index 2245c05d3..da2c6271b 100644 --- a/crates/gpu-gates/src/bin/gate_mimo2_serve.rs +++ b/crates/gpu-gates/src/bin/gate_mimo2_serve.rs @@ -892,8 +892,9 @@ mod gate { let mut inputs = PlanInputs::describe(&split)?; inputs.room = (PLAN_ROOM, HostRead::Given); let place = unset_place(cards).on_host()?; - let batch = - (!place.tier_cards().is_empty()).then(|| bloomery_gpu_mimo2::tier::tier_batch(&inputs)); + let batch = (!place.tier_cards().is_empty()) + .then(|| bloomery_gpu_mimo2::tier::tier_batch(&inputs)) + .transpose()?; let machine = place.machine(None, batch)?(inputs.model.layers); let slots = NonZeroUsize::new(usize::try_from(DEFAULT_SLOTS)?).ok_or("no default slot")?; let plan = diff --git a/crates/gpu-gates/src/bin/gate_musespark_prefill.rs b/crates/gpu-gates/src/bin/gate_musespark_prefill.rs index 6674f694b..5137e5828 100644 --- a/crates/gpu-gates/src/bin/gate_musespark_prefill.rs +++ b/crates/gpu-gates/src/bin/gate_musespark_prefill.rs @@ -530,7 +530,7 @@ mod gate { tier_now: Tier, ) -> Result<(Model, bool, Vec), GateError> { let inputs = PlanInputs::read(&open_split(Arch::MuseSpark, RECIPE)?)?; - let mut two = lcpp_e2e::Arm::Bp.machine(hp.n_layer, place::tier_batch(hp))?; + let mut two = lcpp_e2e::Arm::Bp.machine(hp.n_layer, place::tier_batch(hp)?)?; place::stage_placed(&mut two)?; let ctx = u64::try_from(ctx)?; // The stage card alone, the reference's machine: the budget reads its unbudgeted plan, which a plan of the two diff --git a/crates/gpu-gates/src/bin/gate_solar2_twocard.rs b/crates/gpu-gates/src/bin/gate_solar2_twocard.rs index c32637070..9bcd34bc7 100644 --- a/crates/gpu-gates/src/bin/gate_solar2_twocard.rs +++ b/crates/gpu-gates/src/bin/gate_solar2_twocard.rs @@ -313,7 +313,7 @@ mod gate { let layers = inputs.model.layers; let bp = Place::Bp .on_host()? - .machine(None, Some(place::tier_batch(&inputs.hp)))?; + .machine(None, Some(place::tier_batch(inputs)?))?; let two = bp(layers); let plan = inputs.plan(&two, u64::try_from(CTX)?, &cfg.place)?; let budget = cfg diff --git a/crates/gpu-gates/src/bin/generate_glm5next.rs b/crates/gpu-gates/src/bin/generate_glm5next.rs index 96f430ee2..a6d0b08f2 100644 --- a/crates/gpu-gates/src/bin/generate_glm5next.rs +++ b/crates/gpu-gates/src/bin/generate_glm5next.rs @@ -799,7 +799,7 @@ mod cli { Some(lane_at(placement)?) }; let place = placement.name(); - let tier_batch = glm_place::tier_batch(placement, &inputs); + let tier_batch = glm_place::tier_batch(placement, &inputs)?; let machine = glm_machine( placement, tier_batch, diff --git a/crates/gpu-gates/src/bin/shared/glm5next_place.rs b/crates/gpu-gates/src/bin/shared/glm5next_place.rs index beccc1cab..39fa78350 100644 --- a/crates/gpu-gates/src/bin/shared/glm5next_place.rs +++ b/crates/gpu-gates/src/bin/shared/glm5next_place.rs @@ -39,10 +39,12 @@ pub fn parse(v: &str) -> Result { } /// The expert tier's prompt-batch bytes `place`'s plan reserves for the file -/// `inputs` describes (`model::arch::glm5next::place::tier_batch`), which -/// `Place::machine` takes; `None` under a placement with no tier card. -pub fn tier_batch(place: Place, inputs: &PlanInputs) -> Option { - (!place.tier_cards().is_empty()).then(|| model::arch::glm5next::place::tier_batch(inputs)) +/// described by `inputs` (`model::arch::glm5next::place::tier_batch`), +/// which `Place::machine` takes; `None` under a placement with no tier card. +pub fn tier_batch(place: Place, inputs: &PlanInputs) -> Result, GateError> { + Ok((!place.tier_cards().is_empty()) + .then(|| model::arch::glm5next::place::tier_batch(inputs)) + .transpose()?) } /// The experts `place`'s tier cards hold in its plan at `ctx` positions @@ -63,7 +65,7 @@ pub fn tier_experts( ) -> Result { let machine = glm_machine( place, - tier_batch(place, inputs), + tier_batch(place, inputs)?, slots, checkpoint_price(inputs)?, ctx, diff --git a/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs b/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs index 3b8425314..cbf918642 100644 --- a/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs +++ b/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs @@ -111,7 +111,7 @@ pub fn run(path: &Path, levers: &bloomery_levers::Levers) -> Result Result workstation::placed_with_arena(*stage, layers, arena), [stage, tier] => { let batch = match inputs { - Inputs::Qwen3(i) => q3::tier_batch(&i.hp), - Inputs::Qwen35(i) => q35::leg_tier_batch(&i.spec), - }; + Inputs::Qwen3(i) => q3::tier_batch(i), + Inputs::Qwen35(i) => q35::leg_tier_batch(i), + }?; let mut machine = workstation::plan_tiers(layers, *stage, &[*tier], None, batch)?; q3::stage_placed(&mut machine, arena); machine @@ -2099,7 +2099,7 @@ pub(super) mod tests { /// A dense qwen35 file (no router) plans through [`PlaceQ3::qwen35_on`]: its machine counts /// the whole load's ubatch arena, as the load sizes it, not a placed arena, which a file with - /// no router refuses; and the plan answers at the load's context. + /// no router refuses; an expert tier is refused by name through the same consumer. #[allow( dead_code, reason = "`generate_qwen3moe`'s test calls it; every other bin that includes this file leaves it unused" @@ -2137,6 +2137,27 @@ pub(super) mod tests { assert_eq!(workstation::counted_arena_bytes(&q.machine.cards[0]), arena); q.plan(o.ctx, &PlanLevers::default()) .unwrap_or_else(|e| panic!("the dense plan at {}: {e}", o.ctx)); + let tier = DeviceInfo { + ordinal: 1, + name: "NVIDIA RTX 3090".to_owned(), + total_bytes: 25_769_803_776, + free_bytes: 25_000_000_000, + uuid: [2; 16], + pci_bus: "0000:81:00.0".to_owned(), + held_by: None, + integrated: false, + }; + let census = [census.into_iter().next().expect("the stage card"), tier]; + let two = Place::parse("cuda0+1").expect("two card words"); + let refusal = + match PlaceQ3::qwen35_on(&split, two, &census, o, Seqs::ONE, Beside::default()) { + Err(e) => e, + Ok(_) => panic!("a dense model cannot use an expert tier"), + }; + assert!(matches!( + refusal.downcast_ref::(), + Some(PlacementError::Metadata { key, .. }) if key == "Qwen3.6 expert tier" + )); std::fs::remove_file(&path).expect("the header file"); } } diff --git a/crates/gpu-gates/src/bin/shared/routed_legs/glm5next.rs b/crates/gpu-gates/src/bin/shared/routed_legs/glm5next.rs index a9c0e2b89..5db7eebca 100644 --- a/crates/gpu-gates/src/bin/shared/routed_legs/glm5next.rs +++ b/crates/gpu-gates/src/bin/shared/routed_legs/glm5next.rs @@ -172,7 +172,7 @@ fn machine_of( slots: usize, ) -> Result { let price = checkpoint_price(inputs)?; - let tier_batch = glmplace::tier_batch(place, inputs); + let tier_batch = glmplace::tier_batch(place, inputs)?; Ok(glm_machine(place, tier_batch, slots, price, ctx)?( inputs.model.layers, )) @@ -242,7 +242,6 @@ fn place_of(p: &PlaceError) -> Outcome { PlaceError::Placement(e) => return refused_of(e), PlaceError::PastOracle { .. } => "past-oracle", PlaceError::FrontOver { .. } => "front-over", - PlaceError::TierLeg { .. } => "tier-leg", PlaceError::Broken(_) | PlaceError::NextnCards { .. } | PlaceError::Slots { .. } => { return Outcome::Red(p.to_string()); } diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs b/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs index a982330a6..f670c06ff 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs @@ -503,7 +503,7 @@ impl Placed { (set, slots): (Option, usize), (images, side): (&Images<'_>, Option<&Side>), ) -> Result { - let tier_batch = glm_place::tier_batch(place, inputs); + let tier_batch = glm_place::tier_batch(place, inputs)?; // Every machine reserves its checkpoints at the slot context it is // laid for: the choice's probes each at their own, the load's at the // context it is granted; and the encoder's reserve, as the printed plan does. diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs b/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs index e66507307..495f265e3 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs @@ -78,12 +78,16 @@ pub trait Family: Sized + 'static { /// The expert tier's prompt-batch bytes the plan reserves on `place`'s /// tier card ([`Tiered::tier_batch`]); a body that serves no tier card /// reserves none (the default), and its plan refuses the card by name. - fn tier_batch(place: Place, _inputs: &PlanInputs) -> Option { - (!place.tier_cards().is_empty()).then_some(TierBatchBytes { + fn tier_batch( + place: Place, + _inputs: &PlanInputs, + ) -> Result, GateError> { + Ok((!place.tier_cards().is_empty()).then_some(TierBatchBytes { staging: 0, scratch: 0, host: 0, - }) + leg: None, + })) } /// The tags the model's description names for its think span, which the @@ -323,7 +327,10 @@ impl Tiered for Hybrid { F::think_tags(inputs).map(Some) } - fn tier_batch(place: Place, inputs: &PlanInputs) -> Option { + fn tier_batch( + place: Place, + inputs: &PlanInputs, + ) -> Result, GateError> { F::tier_batch(place, inputs) } diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/mimo2.rs b/crates/gpu-gates/src/bin/shared/serve_seats/mimo2.rs index 019fb66a1..1204c8373 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/mimo2.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/mimo2.rs @@ -501,8 +501,10 @@ impl Tiered for Mimo { /// The tier card's prompt-batch bytes ([`bloomery_gpu_mimo2::tier::tier_batch`]), under a /// placement with one. - fn tier_batch(place: Place, inputs: &PlanInputs) -> Option { - (!place.tier_cards().is_empty()).then(|| bloomery_gpu_mimo2::tier::tier_batch(inputs)) + fn tier_batch(place: Place, inputs: &PlanInputs) -> Result, GateError> { + Ok((!place.tier_cards().is_empty()) + .then(|| bloomery_gpu_mimo2::tier::tier_batch(inputs)) + .transpose()?) } fn flags(flag: &str, value: &str, out: &mut Flags) -> Result { diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs b/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs index 6e06cb60b..92f704bf1 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs @@ -501,8 +501,13 @@ impl Tiered for MiniMax { /// The tier's prompt-batch bytes of the file's widths and down forms over the body's prompt /// unit ([`bloomery_gpu_minimaxm3::tier::tier_batch`]), reserved on the tier card and the /// host; `None` on a placement with no tier card. - fn tier_batch(place: Place, inputs: &PlanInputs) -> Option { - (!place.tier_cards().is_empty()).then(|| bloomery_gpu_minimaxm3::tier::tier_batch(inputs)) + fn tier_batch( + place: Place, + inputs: &PlanInputs, + ) -> Result, GateError> { + Ok((!place.tier_cards().is_empty()) + .then(|| bloomery_gpu_minimaxm3::tier::tier_batch(inputs)) + .transpose()?) } /// The tags the model's description names for its think span. diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/musespark.rs b/crates/gpu-gates/src/bin/shared/serve_seats/musespark.rs index 267a9019a..650f28a66 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/musespark.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/musespark.rs @@ -458,8 +458,13 @@ impl Tiered for Muse { /// The tier's prompt-batch bytes of the file's widths /// ([`place::tier_batch`]), reserved on the tier card and the host; `None` /// on a placement with no tier card. - fn tier_batch(place: Place, inputs: &PlanInputs) -> Option { - (!place.tier_cards().is_empty()).then(|| place::tier_batch(&inputs.hp)) + fn tier_batch( + place: Place, + inputs: &PlanInputs, + ) -> Result, GateError> { + Ok((!place.tier_cards().is_empty()) + .then(|| place::tier_batch(&inputs.hp)) + .transpose()?) } /// The ring is kept out of the expert budget where the common rule starts diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/solar2.rs b/crates/gpu-gates/src/bin/shared/serve_seats/solar2.rs index 61af9a5b9..ce13c171c 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/solar2.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/solar2.rs @@ -182,8 +182,13 @@ impl Family for Solar { /// The tier's prompt-batch bytes of the file's widths ([`place::tier_batch`]), reserved on /// the placement's tier card and the host. - fn tier_batch(place: Place, inputs: &PlanInputs) -> Option { - (!place.tier_cards().is_empty()).then(|| place::tier_batch(&inputs.hp)) + fn tier_batch( + place: Place, + inputs: &PlanInputs, + ) -> Result, bloomery_gpu_gates::GateError> { + Ok((!place.tier_cards().is_empty()) + .then(|| place::tier_batch(inputs)) + .transpose()?) } } diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs b/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs index d2ea83afa..766804fc5 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs @@ -252,12 +252,16 @@ pub trait Tiered: Seat + Sized + 'static { /// The expert tier's prompt-batch bytes the plan reserves on `place`'s tier card, for the file /// `inputs` describes; `None` under a placement with no tier card. A body that serves no tier /// card reserves none (the default), and its plan refuses the card by name. - fn tier_batch(place: Place, _inputs: &PlanInputs) -> Option { - (!place.tier_cards().is_empty()).then_some(TierBatchBytes { + fn tier_batch( + place: Place, + _inputs: &PlanInputs, + ) -> Result, GateError> { + Ok((!place.tier_cards().is_empty()).then_some(TierBatchBytes { staging: 0, scratch: 0, host: 0, - }) + leg: None, + })) } /// Whether the plan of a load on `place` keeps the expert stream's ring out of the expert budget @@ -399,7 +403,7 @@ fn tier_experts( levers: &PlanLevers, ctx: Option, ) -> Result { - let machine = S::machine(offer, S::tier_batch(offer, inputs), inputs.model.layers)?; + let machine = S::machine(offer, S::tier_batch(offer, inputs)?, inputs.model.layers)?; let ctx = ctx.map_or(Ok(1), u64::try_from)?; let plan = inputs.plan(&machine, ctx, levers)?; Ok(plan.tier_n_l.iter().flatten().sum()) @@ -522,7 +526,7 @@ pub fn run(args: &[String], side: &side_seat::Flags) -> Result Result, GateErro inputs.room = (PLAN_ROOM, HostRead::Given); // Plan (b′): the stage on the largest card, the next-largest an expert tier under the host tier, holding the tier's // prompt-batch bytes the file's widths give. - let machine = arm.machine(inputs.model.layers, place::tier_batch(&inputs.hp))?; + let machine = arm.machine(inputs.model.layers, place::tier_batch(&inputs)?)?; let place = PlanLevers::from_levers(&levers)?; let plan = inputs.plan(&machine, u64::try_from(ctx)?, &place)?; let mut m = Body::open_placed(split, &plan, &inputs, 0, levers.host())?; diff --git a/crates/gpu-glm5next/src/tier.rs b/crates/gpu-glm5next/src/tier.rs index e540f4ed2..aee07ed13 100644 --- a/crates/gpu-glm5next/src/tier.rs +++ b/crates/gpu-glm5next/src/tier.rs @@ -62,6 +62,7 @@ impl LegFamily for Glm { n_used: hp.n_used, n_embd: hp.n_embd, ff: hp.expert_ff, + program: model::placement::accepts::Program::Glm5nextBody, } } diff --git a/crates/gpu-mimo2/src/card.rs b/crates/gpu-mimo2/src/card.rs index cf71fb1d1..506e518d3 100644 --- a/crates/gpu-mimo2/src/card.rs +++ b/crates/gpu-mimo2/src/card.rs @@ -27,6 +27,7 @@ fn stacks(l: usize) -> [String; 3] { /// MiMo's card experts: 256 routed experts, 8 a token. const FAMILY: CardFamily = CardFamily { what: "mimo2 CardExperts", + program: model::placement::accepts::Program::Mimo2Body, n_expert: N_EXPERT, n_used: N_USED, stacks, diff --git a/crates/gpu-mimo2/src/tier.rs b/crates/gpu-mimo2/src/tier.rs index 748c7889e..eaded5ed9 100644 --- a/crates/gpu-mimo2/src/tier.rs +++ b/crates/gpu-mimo2/src/tier.rs @@ -34,8 +34,8 @@ use gguf::Split; use model::ModelError; use model::arch::mimo2::place::{self, PlanInputs}; use model::moe::HostLayer; -use model::placement::ModelTensors; use model::placement::workstation::TierBatchBytes; +use model::placement::{ModelTensors, PlacementError}; use model::r8file::R8Source; use models::LayerSpec; @@ -56,8 +56,7 @@ pub const CHUNK: usize = place::TIER_CHUNK as usize; /// The tier's prompt-batch bytes the plan reserves on the tier card and the host for the file /// `inputs` describes: [`place::tier_batch`] over blocks of a prompt unit's rows ([`T_MAX`]), the /// columns the load makes the tier's batch legs for. -#[must_use] -pub fn tier_batch(inputs: &PlanInputs) -> TierBatchBytes { +pub fn tier_batch(inputs: &PlanInputs) -> Result { place::tier_batch(&inputs.hp, T_MAX as u64) } @@ -91,6 +90,7 @@ impl LegFamily for Mimo2 { n_used: hp.n_used, n_embd: hp.n_embd, ff: hp.expert_ff, + program: model::placement::accepts::Program::Mimo2Body, } } diff --git a/crates/gpu-minimaxm3/src/ffn.rs b/crates/gpu-minimaxm3/src/ffn.rs index 828b0897d..3850f6ba4 100644 --- a/crates/gpu-minimaxm3/src/ffn.rs +++ b/crates/gpu-minimaxm3/src/ffn.rs @@ -20,6 +20,7 @@ pub use bloomery_gpu_deepseek41::shared_routed::{ /// The card experts' family: the file's stack names and the router's counts. pub const CARD: CardFamily = CardFamily { what: "minimaxm3 CardExperts", + program: model::placement::accepts::Program::MiniMaxM3Body, n_expert: N_EXPERT, n_used: N_USED, stacks: names::routed_stacks, diff --git a/crates/gpu-minimaxm3/src/tier.rs b/crates/gpu-minimaxm3/src/tier.rs index c591db72c..ca916c671 100644 --- a/crates/gpu-minimaxm3/src/tier.rs +++ b/crates/gpu-minimaxm3/src/tier.rs @@ -20,8 +20,8 @@ use gguf::Split; use model::arch::minimaxm3::host as minimax_host; use model::arch::minimaxm3::place::{self, PlanInputs}; use model::moe::HostLayer; -use model::placement::ModelTensors; use model::placement::workstation::TierBatchBytes; +use model::placement::{ModelTensors, PlacementError}; use model::r8file::R8Source; use models::{LayerSpec, Moe}; @@ -38,8 +38,7 @@ pub const CHUNK: usize = place::TIER_CHUNK as usize; /// The tier's prompt-batch bytes the plan reserves on the tier card and the host for the file /// `inputs` describes: [`place::tier_batch`] over blocks of a prompt unit's rows ([`T_MAX`]), the /// columns the load makes the tier's batch legs for. -#[must_use] -pub fn tier_batch(inputs: &PlanInputs) -> TierBatchBytes { +pub fn tier_batch(inputs: &PlanInputs) -> Result { place::tier_batch(inputs, T_MAX as u64) } @@ -73,6 +72,7 @@ impl LegFamily for MiniMax { n_used: experts.n_used, n_embd: inputs.hp.n_embd, ff: experts.ff, + program: model::placement::accepts::Program::MiniMaxM3Body, } } diff --git a/crates/gpu-solar2/src/ffn.rs b/crates/gpu-solar2/src/ffn.rs index a0f1589d5..7be370e0a 100644 --- a/crates/gpu-solar2/src/ffn.rs +++ b/crates/gpu-solar2/src/ffn.rs @@ -17,6 +17,7 @@ pub use bloomery_gpu_deepseek41::shared_routed::{GATED_FROM, RoutedDims, back, f /// The card experts' family: the file's stack names and the router's counts. pub const CARD: CardFamily = CardFamily { what: "solar2 CardExperts", + program: model::placement::accepts::Program::Solar2Body, n_expert: N_EXPERT, n_used: N_USED, stacks: names::routed_stacks, diff --git a/crates/gpu-solar2/src/tier.rs b/crates/gpu-solar2/src/tier.rs index 071ce8d20..e23773d93 100644 --- a/crates/gpu-solar2/src/tier.rs +++ b/crates/gpu-solar2/src/tier.rs @@ -64,6 +64,7 @@ impl LegFamily for Solar2 { n_used: experts.n_used, n_embd: inputs.hp.n_embd, ff: experts.ff, + program: model::placement::accepts::Program::Solar2Body, } } diff --git a/crates/gpu/src/arch/musespark/body/tier.rs b/crates/gpu/src/arch/musespark/body/tier.rs index 724730577..0c56518f0 100644 --- a/crates/gpu/src/arch/musespark/body/tier.rs +++ b/crates/gpu/src/arch/musespark/body/tier.rs @@ -87,6 +87,7 @@ impl LegFamily for MuseSpark { n_used: hp.n_used, n_embd: hp.latent, ff: hp.expert_width(), + program: model::placement::accepts::Program::MuseSparkBody, } } diff --git a/crates/gpu/src/arch/qwen3moe/body38.rs b/crates/gpu/src/arch/qwen3moe/body38.rs index 30a72d50c..3998e05d4 100644 --- a/crates/gpu/src/arch/qwen3moe/body38.rs +++ b/crates/gpu/src/arch/qwen3moe/body38.rs @@ -2004,7 +2004,7 @@ impl Body38 { hybrid.prepare_batch(gpu.context(), host_cols)?; if tiered { let cards: Vec = tiers.iter().map(|t| t.card).collect(); - hybrid.check_tier_reserves(plan.machine, &cards)?; + hybrid.check_tier_reserves(plan, &cards)?; } // The PLE table's tier is the plan's: the host set built above holds // it on the host; on the NVMe tier its reader reads it. diff --git a/crates/gpu/src/arch/qwen3moe/tier_leg.rs b/crates/gpu/src/arch/qwen3moe/tier_leg.rs index 39915e195..de8854b21 100644 --- a/crates/gpu/src/arch/qwen3moe/tier_leg.rs +++ b/crates/gpu/src/arch/qwen3moe/tier_leg.rs @@ -22,7 +22,7 @@ use gguf::Split; use model::arch::models::{Act as SpecAct, LayerSpec, Moe}; use model::moe::HostLayer; use model::placement::ModelTensors; -use model::placement::accepts::{self, LegArms}; +use model::placement::accepts::Program; use model::r8file::R8Source; use crate::fault::{FaultSink, FaultSite}; @@ -51,14 +51,6 @@ const _: () = assert!(model::arch::qwen35moe::place::LEG_TIER_CHUNK == LEG_CHUNK /// Threads per block of both join entries. const THREADS: u32 = 256; -/// A gate or up stack the placed leg does not run. -const GATE_UP: &str = "a resident routed gate or up stack of a type the placed leg runs \ - (`accepts::prompt_leg_gate_up`)"; - -/// A down stack the placed leg does not run. -const DOWN: &str = - "a resident routed down stack of a type the placed leg runs (`accepts::prompt_leg_down`)"; - /// The routed activation the tier runs: `silu(g)·u`. A clamped SwiGLU, and /// any other activation, is refused by name. fn qwen_act(moe: &Moe) -> Result { @@ -78,19 +70,6 @@ fn qwen_act(moe: &Moe) -> Result { } } -/// The placed leg's arms ([`accepts::leg_arms`] of [`accepts::Program::Qwen3moeBody`]). -/// -/// # Panics -/// -/// When that program names no card-leg arms. -fn qwen_arms() -> (LegArms, (&'static str, &'static str)) { - let arms = match accepts::leg_arms(accepts::Program::Qwen3moeBody) { - Some(arms) => arms, - None => panic!("{WHAT}: Program::Qwen3moeBody names no card-leg arms"), - }; - (arms, (GATE_UP, DOWN)) -} - /// The host views of the routed layers `run`, the refusal naming the run as /// `word`'s. `n_layer` is the description's layer count, so a trunk layer is /// not refused for sitting past a next-token block. @@ -154,6 +133,7 @@ impl LegFamily for Qwen3 { n_used: e.n_used, n_embd: inputs.hp.n_embd, ff: e.ff, + program: Program::Qwen3moeBody, } } @@ -174,10 +154,6 @@ impl LegFamily for Qwen3 { fn tier_act(moe: &Moe) -> Result { qwen_act(moe) } - - fn tier_arms() -> Option<(LegArms, (&'static str, &'static str))> { - Some(qwen_arms()) - } } /// Qwen3.6's facts for the common tier node service. The description's @@ -215,6 +191,7 @@ impl LegFamily for Qwen35 { n_used: moe.top_k as usize, n_embd: inputs.spec.hidden as usize, ff: moe.expert_ff as usize, + program: Program::Qwen35Body, } } @@ -235,10 +212,6 @@ impl LegFamily for Qwen35 { fn tier_act(moe: &Moe) -> Result { qwen_act(moe) } - - fn tier_arms() -> Option<(LegArms, (&'static str, &'static str))> { - Some(qwen_arms()) - } } /// Qwen3-30B's tier node service ([`leg_tier::node_family`]). diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index 8ca374858..13b6365b0 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -41,12 +41,12 @@ use std::mem::ManuallyDrop; use cuda_core::{CudaStream, DeviceBuffer}; use model::arch::models::{Ffn, LayerSpec}; - use model::arch::place_hybrid::RingForms; +use model::placement::accepts::{self, LegArms, Program}; use model::placement::kernels::{self, DownAct, DownForms}; use crate::card_leg::{ - self, CardLeg, LegBufs, LegRun, LegShape, LegStacks, LegSum, RouteBlock, RouteW, + self, CardLeg, LegBufs, LegRun, LegShape, LegStacks, LegSum, RouteBlock, RouteW, STACK_ROLES, }; use crate::card_slots::{CardSlotKernels, Places}; use crate::gemm::Gemm32Kernels; @@ -68,6 +68,8 @@ pub struct CardFamily { pub n_used: usize, /// Layer `l`'s gate, up and down stacks' names. pub stacks: fn(usize) -> [String; 3], + /// The program whose selected gate·up and down launches run the stacks. + pub program: Program, } /// The kernels a run launches: the card leg's (its sum stage included), and @@ -75,6 +77,7 @@ pub struct CardFamily { struct CardKernels { leg: CardLeg, places: CardSlotKernels, + arms: LegArms, } impl CardFamily { @@ -266,8 +269,7 @@ pub struct CardExperts { ff: usize, /// The model's width. n: usize, - /// The forms the card layers' downs read, which every run's buffers hold - /// ([`card_downs`]). + /// The union of the resident layers' selected down activation forms. downs: DownForms, /// The one-token step's buffers. step: CardBufs, @@ -322,6 +324,10 @@ impl CardExperts { tiers: bool, ) -> Result { let what = family.what; + let arms = accepts::leg_arms(family.program).ok_or(GpuError::state( + what, + "the selected program's card leg arms", + ))?; let mut per = Vec::with_capacity(layers.len()); let mut tier_k = Vec::with_capacity(layers.len()); let mut downs = DownForms::NONE; @@ -362,7 +368,7 @@ impl CardExperts { per.push(None); continue; } - let stacks = LegStacks::of(w, layer.names())?; + let stacks = LegStacks::of_arms(w, layer.names(), arms, STACK_ROLES)?; stacks.check(n_card, family.shape(n, ff, 1))?; downs = downs.with(down_act(what, &layer, stacks.down_ty())?); per.push(Some(layer)); @@ -373,6 +379,7 @@ impl CardExperts { k: CardKernels { leg: CardLeg::load(gpu)?, places: CardSlotKernels::load(gpu.context())?, + arms, }, layers: per, tier: tier_k, @@ -714,7 +721,7 @@ impl CardExperts { sel, ) }; - let stacks = LegStacks::of(w, layer.names())?; + let stacks = LegStacks::of_arms(w, layer.names(), self.k.arms, STACK_ROLES)?; places(map.buf(), row_off, &mut *sel)?; let card = LegRun { layer: l, @@ -804,7 +811,7 @@ impl CardKernels { io: &Io<'_>, sel: &DeviceBuffer, ) -> Result<(), GpuError> { - let stacks = LegStacks::of(w, layer.names())?; + let stacks = LegStacks::of_arms(w, layer.names(), self.arms, STACK_ROLES)?; let run = LegRun { layer: l, stacks: &stacks, @@ -830,7 +837,415 @@ impl CardKernels { #[cfg(test)] mod tests { - use super::{DownAct, DownForms, card_downs}; + use super::*; + use gguf::GgmlType; + use gguf::write::{Layout, TensorDecl, Writer}; + use model::arch::models::{self, Mixer, Moe, Residual, Router, RouterNorm, Score}; + use model::placement::{ + Device, ExpertList, ExpertPart, Format, ModelTensor, ModelTensors, Role, RoutedRule, Row, + Segment, + }; + + use crate::card_leg::tests::stack_bytes; + use crate::host::leg_tier::LegTier; + use crate::host::tier::{TierBlock, TierExperts, TierInput, TierIo, TierSet}; + use crate::hybrid::HOST; + + const N: usize = 256; + const FF: usize = 256; + const USED: usize = 8; + const EXPERTS: usize = 16; + const CARD: usize = 4; + const CAP: usize = 3; + const FAMILY: CardFamily = CardFamily { + what: "mixed card wrapper", + n_expert: EXPERTS, + n_used: USED, + stacks: model::arch::names::routed_stacks, + program: Program::Qwen35Body, + }; + + fn layers(n: usize) -> Vec { + vec![ + LayerSpec { + mixer: Mixer::None, + ffn: Ffn::Moe(Moe { + experts: EXPERTS as u32, + top_k: USED as u32, + expert_ff: FF as u32, + latent: None, + pre_exp_norm: false, + post_exp_norm: None, + act: models::Act::SwiGlu { limit: None }, + router: Router { + score: Score::Sigmoid, + bias: false, + norm: RouterNorm::SelectedSum, + scale: 1.0, + hash: false, + }, + shared: None, + }), + residual: Residual::Plain, + extras: Vec::new(), + }; + n + ] + } + + fn tier_act(moe: &Moe) -> Result { + Ok(Act::SwigluClamp { + limit: moe.act.swiglu_limit(), + }) + } + + fn weights(gpu: &Gpu, triples: &[[GgmlType; 3]]) -> Weights { + let mut decls = Vec::new(); + let mut data = Vec::new(); + let mut tensors = Vec::new(); + for (l, triple) in triples.iter().enumerate() { + for (part, (name, &ty)) in (FAMILY.stacks)(l).into_iter().zip(triple).enumerate() { + let (rows, k) = if part == 2 { (N, FF) } else { (FF, N) }; + let bytes = stack_bytes(ty, CARD * rows, k, 7 + (3 * l + part) as u64); + tensors.push(ModelTensor { + name: name.clone(), + shard: 0, + layer: Some(l), + role: Role::RoutedExperts, + ty, + dims: vec![k as u64, rows as u64, CARD as u64], + file_bytes: bytes.len() as u64, + gathered_rows: None, + }); + decls.push(TensorDecl { + name, + dims: vec![k as u64, rows as u64, CARD as u64], + type_id: ty.as_u32(), + nbytes: bytes.len() as u64, + }); + data.push(bytes); + } + } + static CALLS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + let call = CALLS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!( + "bloomery-mixed-wrappers-{}-{call}.gguf", + std::process::id() + )); + let layout = Layout::new(&[], decls.clone()).expect("a layout"); + let file = std::io::BufWriter::new(std::fs::File::create(&path).expect("a file")); + let mut writer = Writer::new(file, layout).expect("a writer"); + for (decl, bytes) in decls.iter().zip(data) { + writer.tensor(&decl.name, &bytes).expect("a tensor"); + } + writer.finish().expect("a finished file"); + let file = gguf::Split::open(&path).expect("the file"); + // IQ3_S has a selected expert row arm, not a generic device format. + let model = ModelTensors { + tensors, + layers: triples.len(), + experts: EXPERTS as u64, + experts_used: USED as u64, + bf16_words: |_| false, + }; + let RoutedRule::Leg { gate_up, down, .. } = accepts::routed_rule(FAMILY.program) else { + panic!("the fixture family must select a card leg"); + }; + let rows = model + .tensors + .iter() + .enumerate() + .map(|(i, t)| { + let format = match ExpertPart::of_name(&t.name).expect("a routed part") { + ExpertPart::Gate | ExpertPart::Up => gate_up(t.ty), + ExpertPart::Down => down(t.ty), + } + .expect("the selected arm's format"); + let mut segment = Segment { + device: Device::Card(0), + format: Format::Card(format), + experts: Some(ExpertList::prefix(CARD as u64).expect("the card prefix")), + resident_bytes: 0, + }; + segment.resident_bytes = segment + .buffer_bytes(t, model.experts_of(t)) + .expect("the selected format's buffers") + .into_iter() + .sum(); + Row { + tensor: i, + segments: vec![segment], + read_bytes: t.file_bytes, + stage: 0, + } + }) + .collect::>(); + let weights = Weights::load_rows(gpu.stream(), &file, &model, &rows, 0) + .expect("resident expert stacks"); + std::fs::remove_file(path).expect("remove the fixture"); + weights + } + + fn assert_bits(got: &[f32], want: &[f32]) { + assert_eq!(got.len(), want.len()); + for (i, (&g, &w)) in got.iter().zip(want).enumerate() { + assert!(g.is_finite() && w.is_finite(), "lane {i}: {g} vs {w}"); + assert_eq!(g.to_bits(), w.to_bits(), "lane {i}: {g} vs {w}"); + } + } + + /// Each real wrapper uses its resident layers' union, not a model-wide + /// quant: both one-form loads and mixed loads conserve bytes and execute + /// step, pass, batch and tier chunks as the same common leg. Host slots + /// stay untouched on the tier, including a chunk's short final run. + #[test] + #[ignore = "needs a CUDA device; `just gate-gpu-lib` runs it on the box"] + fn hw_mixed_card_and_tier_wrappers_conserve_forms_and_rows() { + use GgmlType::{IQ3_S, IQ4_XS, Q5_K, Q6_K, Q8_0}; + let gpu = Gpu::with_device(0).expect("a Gpu on device 0"); + let stream = gpu.stream(); + let leg = CardLeg::load(&gpu).expect("the reference leg"); + for triples in [ + vec![[IQ3_S, IQ3_S, Q5_K], [IQ4_XS; 3]], + vec![[Q5_K, Q5_K, Q8_0]], + vec![[Q5_K, Q5_K, Q6_K]], + vec![ + [IQ3_S, IQ3_S, Q5_K], + [Q5_K, Q5_K, Q8_0], + [Q5_K, Q5_K, Q6_K], + [IQ4_XS; 3], + ], + ] { + let w = weights(&gpu, &triples); + let specs = layers(triples.len()); + let map = SlotMap::prefix(0..triples.len(), EXPERTS, CARD).expect("a map"); + let mut card = CardExperts::new(&gpu, &w, FAMILY, &specs, &map, (N, FF), CAP) + .expect("the card wrapper"); + let mut batch = card.batch(&gpu, CAP).expect("the batch"); + assert_eq!(card.batch_bytes(CAP), batch.bytes()); + let forms = triples.iter().fold(DownForms::NONE, |forms, tys| { + forms.with(kernels::down(tys[2]).expect("a down").1) + }); + let counted = |cap| { + kernels::leg_bytes(cap as u64, USED as u64, N as u64, FF as u64, forms) as usize + }; + assert_eq!( + card.bytes(), + counted(1) + counted(CAP) + 2 * size_of::() * (1 + CAP) * N, + ); + assert_eq!( + batch.bytes(), + counted(CAP) + 2 * size_of::() * CAP * N + size_of::() * CAP * USED, + ); + let shape = FAMILY.shape(N, FF, 2); + let set = TierSet::new( + 0..triples.len(), + EXPERTS, + vec![(0..CARD as u32).collect(); triples.len()], + ) + .expect("the tier set"); + let mut tier = LegTier::new(&gpu, &specs, &set, &w, shape, FAMILY.program, tier_act) + .expect("the tier wrapper"); + assert_eq!(tier.block_bytes(), counted(shape.cap)); + let host_x: Vec = (0..CAP * N).map(|i| (i % 71) as f32 / 71.0 - 0.5).collect(); + let x = DeviceBuffer::from_host(stream, &host_x).expect("the rows"); + let base: Vec = (0..CAP * N).map(|i| (i % 13) as f32 * 0.125).collect(); + let x1 = DeviceBuffer::from_host(stream, &base).expect("the base"); + let route = DeviceBuffer::from_host(stream, &[0.125f32; CAP * USED]).expect("weights"); + let host_sel: Vec = (0..CAP * USED) + .map(|i| { + if i % USED == USED - 1 { + HOST + } else { + (i % CARD) as u32 + } + }) + .collect(); + let sel = DeviceBuffer::from_host(stream, &host_sel).expect("the places"); + let ids = DeviceBuffer::from_host( + stream, + &host_sel + .iter() + .map(|&s| if s == HOST { (EXPERTS - 1) as u32 } else { s }) + .collect::>(), + ) + .expect("the ids"); + let places = DeviceTensor::upload(stream, &map.stage_view(), triples.len(), EXPERTS) + .expect("the map's card view"); + let act = crate::Q8Act::with_tier_cols(stream, CAP, N).expect("the tier block's form"); + for (l, tys) in triples.iter().enumerate() { + let names = (FAMILY.stacks)(l); + let stacks = LegStacks::of_arms( + &w, + names.each_ref().map(String::as_str), + accepts::leg_arms(FAMILY.program).expect("the selected arms"), + STACK_ROLES, + ) + .expect("the reference stacks"); + let mut reference = LegBufs::for_down( + stream, + FAMILY.shape(N, FF, CAP), + kernels::down(tys[2]).expect("a down").1, + ) + .expect("the reference buffers"); + let mut acc = DeviceBuffer::zeroed(stream, CAP * N).expect("the reference sum"); + for m in [1, CAP] { + let io = Io { + normed: &x, + weights: &route, + x1: &x1, + m, + }; + let rule = Act::SwigluClamp { limit: 0.0 }; + leg.enqueue( + &gpu, + &mut reference, + &LegRun { + layer: l, + stacks: &stacks, + normed: &x, + m, + sel: &sel, + n_card: CARD, + rule, + }, + Some(LegSum { + w: &route, + acc: &mut acc, + }), + ) + .expect("the reference"); + let sums = acc.to_host_vec(stream).expect("the reference sums"); + let want: Vec = sums[..m * N] + .iter() + .zip(&base) + .map(|(&a, &b)| a + b) + .collect(); + let got = if m == 1 { + card.enqueue_step(&gpu, &w, l, rule, &io, &sel) + .expect("step"); + card.pre() + } else { + card.enqueue_pass(&gpu, &w, l, rule, &io, &sel) + .expect("pass"); + card.pass_pre().expect("pass output") + }; + assert_bits(&got.to_host_vec(stream).expect("output")[..m * N], &want); + card.enqueue_rows( + &gpu, + &w, + &mut batch, + l, + rule, + &io, + (&ids, &places, l * EXPERTS), + ) + .expect("batch"); + assert_bits( + &batch.pre().to_host_vec(stream).expect("batch output")[..m * N], + &want, + ); + let mut down = + DeviceBuffer::from_host(stream, &vec![1.0e30f32; CAP * USED * N]) + .expect("the tier output"); + if m == 1 { + tier.enqueue_layer( + &gpu, + &w, + l, + TierIo { + act: TierInput::F32(&x), + sel: &sel, + rows: &mut down, + cols: m, + }, + ) + .expect("tier step"); + } else { + tier.enqueue_block( + &gpu, + &w, + l, + TierBlock { + x: &x, + act: &act, + sel: &sel, + cols: m, + down: &mut down, + }, + ) + .expect("tier chunks"); + } + let got = down.to_host_vec(stream).expect("tier output"); + let want = reference + .down() + .to_host_vec(stream) + .expect("reference rows"); + for (s, &place) in host_sel[..m * USED].iter().enumerate() { + if place == HOST { + assert!(got[s * N..(s + 1) * N].iter().all(|&v| v == 1.0e30)); + } else { + assert_bits(&got[s * N..(s + 1) * N], &want[s * N..(s + 1) * N]); + } + } + } + } + assert!(gpu.take_fault().expect("the fault word").is_none()); + } + let set = TierSet::new(0..1, EXPERTS, vec![Vec::new()]).expect("an empty tier"); + let shape = FAMILY.shape(N, FF, 2); + let idle = LegTier::new( + &gpu, + &layers(1), + &set, + &Weights::default(), + shape, + FAMILY.program, + tier_act, + ) + .expect("a configured tier need not hold an expert"); + assert_eq!( + idle.block_bytes(), + kernels::leg_bytes( + shape.cap as u64, + USED as u64, + N as u64, + FF as u64, + DownForms::NONE, + ) as usize, + "an empty tier has no down-activation allocation", + ); + let triple = [IQ3_S, Q5_K, Q8_0]; + let program = Program::Qwen35Body; + let w = weights(&gpu, &[triple]); + let specs = layers(1); + let map = SlotMap::prefix(0..1, EXPERTS, CARD).expect("a map"); + let family = CardFamily { program, ..FAMILY }; + assert!( + matches!( + CardExperts::new(&gpu, &w, family, &specs, &map, (N, FF), CAP), + Err(GpuError::Tensor { .. }), + ), + "{program:?}: {triple:?}" + ); + let set = + TierSet::new(0..1, EXPERTS, vec![(0..CARD as u32).collect()]).expect("a tier set"); + assert!( + matches!( + LegTier::new( + &gpu, + &specs, + &set, + &w, + family.shape(N, FF, 2), + program, + tier_act, + ), + Err(GpuError::Tensor { .. }), + ), + "{program:?}: {triple:?}" + ); + } /// A load's leg buffers hold the forms its card layers' downs read — the /// 32-value form for NVFP4's, both for a mix — and the 128-value form on a diff --git a/crates/gpu/src/card_leg.rs b/crates/gpu/src/card_leg.rs index 6a318423d..12a0138b4 100644 --- a/crates/gpu/src/card_leg.rs +++ b/crates/gpu/src/card_leg.rs @@ -533,15 +533,12 @@ struct LegStack<'a> { } impl<'a> LegStack<'a> { - /// The resident stack `name`: a `KQuant` stack of a type `runs` admits in - /// the stack's role, a Q5_0 stack in the packed layout where `runs` admits - /// Q5_0, or an NVFP4 stack in its planes where `planes` admits NVFP4; - /// `role` is the refusal's words. + /// The resident stack `name` of a type `runs` admits in its role, + /// including plane-backed NVFP4 when admitted; `role` names a refusal. fn of( w: &'a Weights, name: &'a str, runs: fn(GgmlType) -> bool, - planes: fn(GgmlType) -> bool, role: &'static str, ) -> Result, GpuError> { match w.get(name) { @@ -557,7 +554,7 @@ impl<'a> LegStack<'a> { w: StackW::Q5_0(w), k: *k, }), - Some(DevWeight::Nvfp4Stack { k, .. }) if planes(GgmlType::NVFP4) => Ok(LegStack { + Some(DevWeight::Nvfp4Stack { k, .. }) if runs(GgmlType::NVFP4) => Ok(LegStack { name, ty: GgmlType::NVFP4, w: StackW::Nvfp4(w.nvfp4_stack(WHAT, name)?), @@ -629,6 +626,12 @@ impl<'a> LegStack<'a> { const GATE_UP: &str = "a resident routed gate or up stack of a type the card leg runs (`kernels::card_gate_up_runs`)"; +/// The selected program admits the same resident stack roles on card and tier. +pub(crate) const STACK_ROLES: (&str, &str) = ( + "a resident routed gate or up stack the selected program's gate·up arm runs", + "a resident routed down stack the selected program's down arm runs", +); + /// A layer's three resident routed stacks, the gate and up of one type. #[derive(Clone, Copy)] pub struct LegStacks<'a> { @@ -651,8 +654,8 @@ impl<'a> LegStacks<'a> { w, names, LegArms { - gate_up: kernels::card_gate_up_runs, - down: kernels::card_down_runs, + gate_up: accepts::planes_leg_gate_up, + down: accepts::planes_leg_down, }, (GATE_UP, DOWN), ) @@ -669,8 +672,8 @@ impl<'a> LegStacks<'a> { w, names, LegArms { - gate_up: kernels::card_gate_up_runs, - down: accepts::resident_down_runs, + gate_up: accepts::planes_leg_gate_up, + down: |ty| accepts::resident_down_runs(ty) || kernels::card_planes_runs(ty), }, (GATE_UP, DOWN), ) @@ -690,9 +693,9 @@ impl<'a> LegStacks<'a> { ) -> Result, GpuError> { let [gate, up, down] = names; let (gate, up, down) = ( - LegStack::of(w, gate, arms.gate_up, kernels::card_planes_runs, roles.0)?, - LegStack::of(w, up, arms.gate_up, kernels::card_planes_runs, roles.0)?, - LegStack::of(w, down, arms.down, kernels::card_planes_runs, roles.1)?, + LegStack::of(w, gate, arms.gate_up, roles.0)?, + LegStack::of(w, up, arms.gate_up, roles.0)?, + LegStack::of(w, down, arms.down, roles.1)?, ); if up.ty != gate.ty { return Err(GpuError::Tensor { @@ -1512,8 +1515,8 @@ impl<'a> Relu2Stacks<'a> { const DOWN: &str = "a resident routed down stack of a 32-value-block type the card leg runs (`kernels::card_sel32_runs`)"; let [up, down] = names; Ok(Relu2Stacks { - up: LegStack::of(w, up, kernels::card_sel32_runs, |_| false, UP)?, - down: LegStack::of(w, down, kernels::card_sel32_runs, |_| false, DOWN)?, + up: LegStack::of(w, up, kernels::card_sel32_runs, UP)?, + down: LegStack::of(w, down, kernels::card_sel32_runs, DOWN)?, }) } diff --git a/crates/gpu/src/host/batch.rs b/crates/gpu/src/host/batch.rs index 5a69a4707..29dc6c687 100644 --- a/crates/gpu/src/host/batch.rs +++ b/crates/gpu/src/host/batch.rs @@ -29,7 +29,7 @@ use crate::graph::{MappedHost, cu}; use crate::tensor::WindowMut; use crate::{Gpu, GpuError, Q8Act}; use cuda_core::{CudaContext, CudaEvent, CudaStream, DeviceBuffer, PinnedHostBuffer, sys}; -use model::placement::Machine; +use model::placement::Plan; use model::placement::workstation::{TIER_BATCH_HOST_RESERVE, TIER_BATCH_RESERVE}; use model::{Tensor2, Tensor2View}; use std::mem::size_of; @@ -1079,18 +1079,19 @@ impl HostTier { Ok(()) } - /// The expert tiers' prompt-batch bytes against the reserves `machine`'s + /// The expert tiers' prompt-batch bytes against the selected reserves the /// plan carries for them: on each tier's card — tier `t`'s is index - /// `cards[t]` of [`Machine::all_cards`] — its batch staging and the + /// `cards[t]` of [`model::placement::Machine::all_cards`] — its batch staging and the /// architecture's block scratch ([`TierCard::block_bytes`]) against that - /// card's [`TIER_BATCH_RESERVE`] row; on the host, the tier legs' rows and + /// card's selected [`TIER_BATCH_RESERVE`] row in `plan`; on the host, the tier legs' rows and /// places against the host's [`TIER_BATCH_HOST_RESERVE`] row. A card list /// that is not one card a tier, a missing row, a row named twice, or a /// difference is refused by name, naming the reserve. Load-time only, once /// [`HostTier::prepare_batch`] has made the legs. Step-only tiers make /// no batch leg ([`HostTier::prepare_tier_batch`]) and none is checked. - pub fn check_tier_reserves(&self, machine: &Machine, cards: &[usize]) -> Result<(), GpuError> { + pub fn check_tier_reserves(&self, plan: &Plan<'_>, cards: &[usize]) -> Result<(), GpuError> { const WHAT: &str = "HostTier::check_tier_reserves"; + let machine = plan.machine; if self.tiers.iter().any(|t| !t.supports_blocks()) { return Ok(()); } @@ -1124,6 +1125,7 @@ impl HostTier { &card.reserves, TIER_BATCH_RESERVE, port.tier_device_bytes(i) + t.block_bytes()?, + Some(index), )); } rows.push(( @@ -1131,15 +1133,27 @@ impl HostTier { &machine.host().reserves, TIER_BATCH_HOST_RESERVE, port.tier_host_bytes(), + None, )); - for (on, reserves, name, got) in rows { + for (on, reserves, name, got, card) in rows { let want = match reserves .iter() .filter(|(n, _)| n == name) .collect::>() .as_slice() { - [(_, b)] => *b, + [(_, b)] => match card { + Some(c) => plan + .card_reserve(c, name) + .map_err(|e| GpuError::plan(WHAT, e))? + .ok_or_else(|| { + GpuError::shape( + WHAT, + format!("the plan's selected reserve {name:?} on {on} is invalid"), + ) + })?, + None => *b, + }, named => { return Err(GpuError::shape( WHAT, diff --git a/crates/gpu/src/host/leg_tier.rs b/crates/gpu/src/host/leg_tier.rs index 0762476e1..c35afc97d 100644 --- a/crates/gpu/src/host/leg_tier.rs +++ b/crates/gpu/src/host/leg_tier.rs @@ -8,9 +8,9 @@ //! //! The tier's layer is the card leg ([`CardLeg::enqueue_rows`]) over the //! tier's stacks: the q8_1 of the activation, the gate·up `_sel` with the -//! routed activation the family names (clamped SwiGLU unless it names -//! another), the q8_1 of the tier slots' columns and the down -//! `_sel` into the tier's rows; over a prompt batch's block, the same run +//! routed activation the family names, the selected down form of the tier +//! slots' columns and the down `_sel` into the tier's rows; over a prompt +//! batch's block, the same run //! chunk by chunk of `shape.cap` tokens. Each slot's down output is the one //! the stage card would write for it, so the card sum over both cards' slots //! in slot order is the one-card sum over their union. The row's tier image @@ -30,8 +30,8 @@ use model::ModelError; use model::arch::models::{Ffn, LayerSpec, Moe}; use model::arch::names::routed_stacks; use model::moe::HostLayer; -use model::placement::accepts::LegArms; -use model::placement::kernels::{self, DownAct, DownForms}; +use model::placement::accepts::{self, LegArms, Program}; +use model::placement::kernels::{self, DownForms}; use model::placement::{ModelTensors, Plan}; use model::r8file::R8Source; @@ -40,7 +40,7 @@ use super::run::{HostRun, HostWidths}; use super::tier::{ TierAct, TierBlock, TierCard, TierExperts, TierInput, TierIo, TierOpen, TierSet, TierShape, }; -use crate::card_leg::{CardLeg, LegBufs, LegRun, LegShape, LegStacks}; +use crate::card_leg::{CardLeg, LegBufs, LegRun, LegShape, LegStacks, STACK_ROLES}; use crate::hybrid::{Hybrid, SlotMap}; use crate::kquant::Act; use crate::model::CardBytes; @@ -173,9 +173,8 @@ impl TierLayer { /// leg over the tier's stacks into the tier's rows; over a prompt batch's /// block, the same chunk by chunk. Its scratch, one [`LegBufs`] for /// `shape.cap` tokens that a step's one token and each chunk share, is made -/// at load. The activation and the stack arms are the family's -/// ([`LegFamily::tier_act`], [`LegFamily::tier_arms`]); a family that names -/// no arms uses the card leg's own. +/// at load. The activation is the family's ([`LegFamily::tier_act`]); the +/// selected program owns the stack arms ([`accepts::leg_arms`]). pub struct LegTier { leg: CardLeg, bufs: LegBufs, @@ -183,30 +182,33 @@ pub struct LegTier { /// Per layer of `layers`, `None` off the tier. cfg: Vec>, shape: LegShape, - /// The family's stack arms, or the card leg's own when it named none. - arms: Option<(LegArms, (&'static str, &'static str))>, + arms: LegArms, } impl LegTier { /// The tier's computation on `gpu` over `set`, whose stacks `w` holds, /// for the description `layers`, through the leg of `shape` (`cap` tokens /// of `slots_per_col` slots a run, `n` wide, experts `ff` wide). The - /// activation is `F`'s ([`LegFamily::tier_act`]) and the stacks are read - /// by `F`'s arms ([`LegFamily::tier_arms`]; the card leg's own when the - /// family names none). A tier layer whose three stacks are absent, of a - /// type those arms do not read, or of other rows than its experts', and - /// a layer off the tier with resident stacks, are refused by name. The + /// activation is the family's `act` ([`LegFamily::tier_act`]) and the + /// stacks are read by the selected `program`'s arms. Missing or unreadable + /// stacks, rows that differ from the tier's experts, and resident stacks + /// on a layer off the tier are refused by name. The /// leg's buffers hold the columns in the forms the tier layers' downs - /// read ([`kernels::down`]), the 128-value form on a tier of no layer. + /// read ([`kernels::down`]), no down form on a tier of no layer. /// Load-time only. - pub fn new( + pub fn new( gpu: &Gpu, layers: &[LayerSpec], set: &TierSet, w: &Weights, shape: LegShape, + program: Program, + act: fn(&Moe) -> Result, ) -> Result { - let arms = F::tier_arms(); + let arms = accepts::leg_arms(program).ok_or(GpuError::state( + WHAT, + "the selected program's card leg arms", + ))?; let run = set.layers(); let mut cfg = Vec::with_capacity(run.len()); let mut downs = DownForms::NONE; @@ -232,10 +234,10 @@ impl LegTier { }; let layer = TierLayer { k, - rule: F::tier_act(m)?, + rule: act(m)?, names, }; - let stacks = stacks_of(arms, w, layer.names())?; + let stacks = LegStacks::of_arms(w, layer.names(), arms, STACK_ROLES)?; stacks.check(k, shape)?; let (_, form) = kernels::down(stacks.down_ty()).ok_or_else(|| GpuError::Tensor { what: WHAT, @@ -245,9 +247,6 @@ impl LegTier { downs = downs.with(form); cfg.push(Some(layer)); } - if downs == DownForms::NONE { - downs = DownForms::of(DownAct::Q8Act); - } Ok(LegTier { leg: CardLeg::load(gpu)?, bufs: LegBufs::for_downs(gpu.stream(), shape, downs)?, @@ -259,19 +258,6 @@ impl LegTier { } } -/// `names` read by `arms`, or by the card leg's own arms when the family -/// named none ([`LegStacks::of`]). -fn stacks_of<'a>( - arms: Option<(LegArms, (&'static str, &'static str))>, - w: &'a Weights, - names: [&'a str; 3], -) -> Result, GpuError> { - match arms { - None => LegStacks::of(w, names), - Some((arms, roles)) => LegStacks::of_arms(w, names, arms, roles), - } -} - /// Layer `layer`'s tier experts of `cfg` (the tier's layers from /// `layers.start`); a layer the tier holds none of is refused by name as /// `what`. @@ -323,7 +309,7 @@ impl TierExperts for LegTier { .. } = *shape; let tl = layer_of(layers, cfg, layer, W)?; - let stacks = stacks_of(arms, weights, tl.names())?; + let stacks = LegStacks::of_arms(weights, tl.names(), arms, STACK_ROLES)?; for c0 in (0..io.cols).step_by(cap) { let cn = cap.min(io.cols - c0); let slots = cn * slots_per_col; @@ -367,7 +353,7 @@ impl TierExperts for LegTier { .to_string(), }); }; - let stacks = stacks_of(arms, weights, tl.names())?; + let stacks = LegStacks::of_arms(weights, tl.names(), arms, STACK_ROLES)?; let run = LegRun { layer, stacks: &stacks, @@ -482,6 +468,8 @@ pub struct LegWidths { pub n_embd: usize, /// A routed expert's width. pub ff: usize, + /// The program whose card leg runs these experts. + pub program: Program, } impl LegWidths { @@ -560,13 +548,6 @@ pub trait LegFamily { limit: moe.act.swiglu_limit(), }) } - - /// The arms a tier layer's stacks are read by, with the gate-or-up - /// refusal's words and the down's, or `None` for the card leg's own - /// arms ([`LegStacks::of`]). - fn tier_arms() -> Option<(LegArms, (&'static str, &'static str))> { - None - } } /// The family `F`'s tier node service: the placement tensors of a file, the @@ -609,13 +590,16 @@ fn node_experts( open: &NodeOpen, ) -> Result, GpuError> { let inputs = node_inputs::(file, open)?; - let shape = F::widths(&inputs).shape(F::CHUNK); - Ok(Box::new(LegTier::new::( + let widths = F::widths(&inputs); + let shape = widths.shape(F::CHUNK); + Ok(Box::new(LegTier::new( gpu, F::layers(&inputs), &open.set, w, shape, + widths.program, + F::tier_act, )?)) } @@ -696,5 +680,5 @@ pub fn hang_tiers( let planned: Vec = hang.tiers.iter().map(|t| t.card).collect(); hybrid.attach_tiers(cards, gpu)?; hybrid.prepare_batch(gpu.context(), hang.cols)?; - hybrid.check_tier_reserves(plan.machine, &planned) + hybrid.check_tier_reserves(plan, &planned) } diff --git a/crates/gpu/src/host/node.rs b/crates/gpu/src/host/node.rs index 8407b206c..12f10ea6d 100644 --- a/crates/gpu/src/host/node.rs +++ b/crates/gpu/src/host/node.rs @@ -321,14 +321,23 @@ impl NodeOpen { .find(|d| self.device.is_some_and(|id| id.uuid == d.uuid)) .ok_or_else(|| GpuError::shape(WHAT, "the projected tier is absent from the census"))?; let machine = workstation::plan_on(spec_of_device(device), self.layers); - let reserve = plan - .machine - .card(self.card) - .ok_or_else(|| { + let card = plan.machine.card(self.card).ok_or_else(|| { + GpuError::shape(WHAT, "the projected tier card is absent from the plan") + })?; + // Host-only projections have no card totals; their fixed reserves + // still belong to the machine. Dynamic scratch needs selected totals. + let reserve = if card.tier_leg.is_some() { + let totals = plan.cards.get(self.card).ok_or_else(|| { GpuError::shape(WHAT, "the projected tier card is absent from the plan") - })? - .set_aside_bytes() - .ok_or_else(|| GpuError::shape(WHAT, "the projected tier reserve overflows"))?; + })?; + totals + .context_bytes + .checked_add(totals.scratch_bytes) + .and_then(|bytes| bytes.checked_add(totals.reserve_bytes)) + } else { + card.set_aside_bytes() + } + .ok_or_else(|| GpuError::shape(WHAT, "the projected tier reserve overflows"))?; let mut projected = model::placement::nodes::routed_host( plan.model, &plan.rows, diff --git a/crates/model/src/arch/deepseek41/kv.rs b/crates/model/src/arch/deepseek41/kv.rs index d7fd1ebb7..8a45d8fd6 100644 --- a/crates/model/src/arch/deepseek41/kv.rs +++ b/crates/model/src/arch/deepseek41/kv.rs @@ -263,6 +263,7 @@ mod tests { let machine = Machine { cards: vec![Card { name: "pool".into(), + tier_leg: None, device: None, usable_bytes: pool, context_bytes: 0, diff --git a/crates/model/src/arch/glm5next/place.rs b/crates/model/src/arch/glm5next/place.rs index 11d05d9e3..f4baa58f6 100644 --- a/crates/model/src/arch/glm5next/place.rs +++ b/crates/model/src/arch/glm5next/place.rs @@ -146,53 +146,22 @@ fn down_forms(downs: impl Iterator) -> DownForms { .fold(DownForms::NONE, |forms, (_, act)| forms.with(act)) } -/// The down forms of the expert tier's leg buffers: those of every routed layer the card experts -/// run ([`carded_layers`]), the tier holding experts of any of them, tiles' layers included (the -/// tier runs the card leg over all of them); the 128-value form when the file has none, the form -/// the load's leg buffers default to. -fn tier_downs(model: &ModelTensors) -> DownForms { - let forms = down_forms(carded_layers(model).map(|(_, [_, _, d])| d)); - if forms == DownForms::NONE { - DownForms::of(DownAct::Q8Act) - } else { - forms - } -} - -/// The first routed layer of `model` the card experts run that the expert -/// tier card's plan does not place on its leg, with its stacks' types: the -/// plan places the file's blocks there over the 128-value form, each part by -/// its arm ([`kernels::card_gate_up_runs`] for the gate and up, -/// [`kernels::card_q8act_down_runs`] for the down, a Q6_K down among them), -/// so a plan that hangs a tier card is refused it ([`PlaceError::TierLeg`]). -fn off_tier_layer(model: &ModelTensors) -> Option<(usize, [GgmlType; 3])> { - carded_layers(model).find(|&(_, [g, u, d])| { - !(kernels::card_gate_up_runs(g) - && kernels::card_gate_up_runs(u) - && kernels::card_q8act_down_runs(d)) - }) -} - /// Tokens a chunk of the tier's prompt-batch service runs at most: the /// stage card's chunk of the card experts' launches, which the body binds /// to its own constant. pub const TIER_CHUNK: u64 = 8; -/// The expert tier's prompt-batch bytes ([`TierBatchBytes`]) for the file `inputs` describes and -/// blocks of the host union's columns: the tier card's staging, one run of the card leg's buffers -/// over [`TIER_CHUNK`] tokens in each form the tier layers' downs read ([`tier_downs`]; the -/// card leg's own formula, [`workstation::tier_leg_batch_bytes_of`]) and the host's rows and -/// places, which the plan reserves on the tier card and the host and the load checks its -/// allocations against. -#[must_use] -pub fn tier_batch(inputs: &PlanInputs) -> TierBatchBytes { +/// The expert tier's prompt-batch bytes for the selected program's routed +/// layers: staging, the common leg's candidate-dependent scratch and the +/// host rows and places. The load checks its allocations against the plan. +pub fn tier_batch(inputs: &PlanInputs) -> Result { let hp = &inputs.hp; - workstation::tier_leg_batch_bytes_of( + workstation::try_tier_leg_batch_bytes_of( hp.n_embd as u64, hp.expert_ff as u64, hp.n_used as u64, (TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64), - tier_downs(&inputs.model), + crate::placement::accepts::leg_down_forms(Program::Glm5nextBody, &inputs.model), ) } @@ -493,16 +462,6 @@ pub enum PlaceError { /// A plan of no resident sequence: a load serves at least one. #[error("a plan of {slots} resident sequences: a load serves at least one")] Slots { slots: usize }, - /// A machine that hangs an expert tier card beside a file with a card - /// layer the plan does not place on the tier's leg (NVFP4's planes, a down - /// over the 32-value form): the plan places the file's blocks over the - /// 128-value form alone there. - #[error( - "layer {layer}: routed stacks {tys:?} run on the stage card's leg, and the expert tier \ - card's leg reads the file's blocks over 128-value q8_1 columns alone; plan it without \ - the tier card" - )] - TierLeg { layer: usize, tys: [GgmlType; 3] }, } impl PlanRefusal for PlaceError { @@ -771,7 +730,6 @@ impl PlanInputs { if slots == 0 { return Err(PlaceError::Slots { slots }); } - self.refuse_off_tier(machine)?; let kv = self.kv.with_lanes(lanes); let terms = self.seq_terms(&kv, None); let n = slots as u64; @@ -817,18 +775,6 @@ impl PlanInputs { } Ok(plan) } - - /// Refused by name ([`PlaceError::TierLeg`]) when `machine` hangs an - /// expert tier card and a card layer of the file is one the tier's leg - /// does not run. - fn refuse_off_tier(&self, machine: &Machine) -> Result<(), PlaceError> { - match off_tier_layer(&self.model) { - Some((layer, tys)) if machine.expert_tiers().next().is_some() => { - Err(PlaceError::TierLeg { layer, tys }) - } - _ => Ok(()), - } - } } /// Card bytes the NextN walk's arena may hold beside the plan's tensors @@ -949,6 +895,7 @@ fn nextn_machine() -> Machine { Machine { cards: vec![Card { name: "nextn".to_string(), + tier_leg: None, device: None, usable_bytes: u64::MAX, context_bytes: 0, @@ -1133,7 +1080,6 @@ impl PlanInputs { if slots == 0 { return Err(PlaceError::Slots { slots }); } - self.refuse_off_tier(machine)?; let kv = self.kv.with_lanes(KdaLanes::Two); let terms = self.seq_terms(&kv, Some(nextn)); let n = slots as u64; @@ -1641,8 +1587,7 @@ mod tests { use super::super::super::hparams::{Hc, Hparams, Indexer}; use super::super::{ DenseForms, KdaLanes, Kind, KvLayout, MixerForms, NextnInputs, NextnKv, PlaceError, - PlanInputs, TIER_CHUNK, card_leg_bytes, card_leg_forms, prompt_reserve_bytes, - row_bytes, tier_batch, + PlanInputs, card_leg_bytes, card_leg_forms, prompt_reserve_bytes, row_bytes, }; use crate::placement::churn::ChurnPool; use crate::placement::workstation::{HostNeed, HostRead}; @@ -1729,6 +1674,7 @@ mod tests { Machine { cards: vec![Card { name: "card".to_string(), + tier_leg: None, device: None, usable_bytes: 1 << 40, context_bytes: 1 << 20, @@ -2095,54 +2041,6 @@ mod tests { } } - /// The expert tier's prompt-batch reserve follows the down forms of the file's card layers: - /// its scratch is the card leg's one formula (`kernels::leg_bytes`, which `LegBufs::for_downs` - /// allocates by) over `TIER_CHUNK` tokens in each form a layer's down reads, the staging and - /// host terms beside it moved by no form. A K-quant file, a Q6_K down and a file with no card - /// layer reserve the 128-value form alone; an NVFP4 layer or a Q8_0 down the 32-value form; layers - /// of both, both, the tiles' layers (Q4_K gate and up over a Q5_K down) counted with the rest. - /// Mutants: the reserve of the 128-value form alone; the tiles' layers left out of the forms. - #[test] - fn the_tier_reserve_follows_the_routed_downs() { - use crate::placement::kernels::{DownAct, DownForms, leg_bytes}; - use GgmlType::{IQ3_S, IQ4_XS, NVFP4, Q4_K, Q5_K, Q6_K, Q8_0}; - let q8 = DownForms::of(DownAct::Q8Act); - let b32 = DownForms::of(DownAct::Q8Blocks32); - let base = tier_batch(&inputs()); - let hp = inputs().hp; - let leg = |forms| { - leg_bytes( - TIER_CHUNK, - hp.n_used as u64, - hp.n_embd as u64, - hp.expert_ff as u64, - forms, - ) - }; - assert_ne!(leg(q8), leg(b32)); - for (layers, forms) in [ - (vec![], q8), - (vec![[Q4_K, Q4_K, Q5_K]], q8), - (vec![[Q5_K, Q5_K, Q6_K]], q8), - (vec![[IQ3_S, IQ3_S, IQ4_XS]], q8), - (vec![[NVFP4; 3]], b32), - (vec![[Q4_K, Q4_K, Q8_0]], b32), - ( - vec![[IQ3_S, IQ3_S, IQ4_XS], [NVFP4; 3]], - q8.with(DownAct::Q8Blocks32), - ), - ( - vec![[Q4_K, Q4_K, Q5_K], [NVFP4; 3]], - q8.with(DownAct::Q8Blocks32), - ), - ] { - let batch = tier_batch(&routed_types(&layers)); - assert_eq!(batch.scratch, leg(forms), "{layers:?}"); - assert_eq!(batch.staging, base.staging, "{layers:?}"); - assert_eq!(batch.host, base.host, "{layers:?}"); - } - } - /// A Q5_K gate and up with a Q6_K down takes every expert on a card /// with room for the stack; a Q6_K gate and up takes none. /// PIN(2026-10-22): the down arm admits Q6_K, so the layer is @@ -2450,44 +2348,74 @@ mod tests { } } - /// A machine that hangs an expert tier card is refused, by layer - /// and types, a file with a card layer the tier's leg does not run - /// (an NVFP4 layer: its planes and 32-value form); without the tier - /// card the same file plans, the NVFP4 layer's experts on the stage - /// card, and a K-quant file is not refused with it - /// ([`node_ranges_use_the_common_loaded_assignment`]). Mutants: the - /// refusal dropped, which leaves the tier's load to refuse the - /// layer's stacks; the body's plan rule left on the common leg's, which - /// keeps the NVFP4 layer on the host, off the stage card's planes. + /// The tier reserves precisely the selected layers' down forms, + /// including a mixed 128/32-value load; a gate/up mismatch remains + /// host-only and cannot add a form. #[test] - fn a_tier_card_is_refused_a_layer_its_leg_does_not_run() { + fn tier_scratch_follows_selected_mixed_layers() { + use super::super::TIER_CHUNK; + use crate::placement::kernels::{self, DownAct, DownForms}; + use GgmlType::{IQ3_S, NVFP4, Q5_K, Q8_0}; let mut inputs = routed(64 << 30); - for t in inputs + inputs .model .tensors - .iter_mut() - .filter(|t| t.role == Role::RoutedExperts && t.layer == Some(1)) - { - t.ty = GgmlType::NVFP4; - t.file_bytes = t.dims.iter().product::() / 64 * 36; - } - let plain = unified(64 << 30); - let mut machine = plain.clone(); - let mut tier = machine.cards[0].clone(); - tier.name = "remote".into(); - tier.head = false; - tier.layers = 0..0; - machine.tiers.push(tier); - match inputs.plan(&machine, CTX, &PlanLevers::default()) { - Err(PlaceError::TierLeg { layer, tys }) => { - assert_eq!((layer, tys), (1, [GgmlType::NVFP4; 3])); + .retain(|t| t.role != Role::RoutedExperts || t.layer.is_some_and(|l| l < 2)); + let set = |inputs: &mut PlanInputs, l, tys: [GgmlType; 3]| { + for t in inputs + .model + .tensors + .iter_mut() + .filter(|t| t.role == Role::RoutedExperts && t.layer == Some(l)) + { + let part = match crate::placement::ExpertPart::of_name(&t.name) { + Some(crate::placement::ExpertPart::Gate) => 0, + Some(crate::placement::ExpertPart::Up) => 1, + Some(crate::placement::ExpertPart::Down) => 2, + None => unreachable!(), + }; + t.ty = tys[part]; + } + }; + for (layers, forms) in [ + ( + [[IQ3_S, IQ3_S, Q5_K], [Q5_K, Q5_K, Q5_K]], + DownForms::of(DownAct::Q8Act), + ), + ( + [[Q5_K, Q5_K, Q8_0], [Q5_K, Q5_K, Q8_0]], + DownForms::of(DownAct::Q8Blocks32), + ), + ( + [[IQ3_S, IQ3_S, Q5_K], [NVFP4; 3]], + DownForms::of(DownAct::Q8Act).with(DownAct::Q8Blocks32), + ), + ( + [[IQ3_S, IQ3_S, Q5_K], [Q5_K, Q5_K, Q8_0]], + DownForms::of(DownAct::Q8Act).with(DownAct::Q8Blocks32), + ), + ( + [[IQ3_S, IQ3_S, Q5_K], [IQ3_S, Q5_K, Q8_0]], + DownForms::of(DownAct::Q8Act), + ), + ] { + for (l, tys) in layers.into_iter().enumerate() { + set(&mut inputs, l, tys); } - other => panic!("{:?}", other.map(|p| p.n_l.clone())), + let bytes = super::super::tier_batch(&inputs).expect("tier bytes"); + let hp = &inputs.hp; + assert_eq!( + bytes.scratch, + kernels::leg_bytes( + TIER_CHUNK, + hp.n_used as u64, + hp.n_embd as u64, + hp.expert_ff as u64, + forms, + ), + "{layers:?}", + ); } - let p = inputs - .plan(&plain, CTX, &PlanLevers::default()) - .expect("the plan without the tier card"); - assert!(p.n_l[1] > 0, "the NVFP4 layer on the card: {:?}", p.n_l); } #[test] diff --git a/crates/model/src/arch/mimo2/place.rs b/crates/model/src/arch/mimo2/place.rs index 14ad7aa30..03e4345ec 100644 --- a/crates/model/src/arch/mimo2/place.rs +++ b/crates/model/src/arch/mimo2/place.rs @@ -119,8 +119,7 @@ pub const TIER_CHUNK: u64 = 16; /// plan's rule carries reads (the rule's down arm is the leg's `Q8Act` downs), and the host's rows /// and places ([`workstation::tier_leg_batch_bytes`]). The plan reserves them on the tier card and /// the host, and the load checks its allocations against them. -#[must_use] -pub fn tier_batch(hp: &Hparams, cols: u64) -> TierBatchBytes { +pub fn tier_batch(hp: &Hparams, cols: u64) -> Result { workstation::tier_leg_batch_bytes( hp.n_embd as u64, hp.expert_ff as u64, @@ -266,18 +265,12 @@ mod tests { let inputs = PlanInputs::describe(&split).expect("the synthetic header describes"); let _ = std::fs::remove_file(&path); let hp = &inputs.hp; - assert_eq!( - tier_batch(hp, COLS), - workstation::tier_leg_batch_bytes( - hp.n_embd as u64, - hp.expert_ff as u64, - hp.n_used as u64, - TIER_CHUNK, - COLS, - ) - ); let routed = runtime::layer::hosted(&inputs.spec.layers).map_or(0, |run| run.len()); - let tiered = workstation::plan_bp(hp.n_trunk, None, tier_batch(hp, COLS)); + let tiered = workstation::plan_bp( + hp.n_trunk, + None, + tier_batch(hp, COLS).expect("valid tier geometry"), + ); let plain = workstation::plan_gate(hp.n_trunk); assert_eq!( inputs.prompt_batch_bytes(&plain).expect("a plain machine"), diff --git a/crates/model/src/arch/minimaxm3/place.rs b/crates/model/src/arch/minimaxm3/place.rs index 6cf405c84..e8528fb48 100644 --- a/crates/model/src/arch/minimaxm3/place.rs +++ b/crates/model/src/arch/minimaxm3/place.rs @@ -22,10 +22,10 @@ use crate::arch::names::routed_stacks; use crate::arch::place_hybrid::{ self, ring_pass_bytes, route_bytes, stream_ring_bytes, tier_places_bytes, wide_bytes, }; -use crate::placement::accepts::planes_leg_down; +use crate::placement::PlacementError; +use crate::placement::accepts::{self, Program}; use crate::placement::kernels::{self, DownAct, DownForms}; use crate::placement::workstation::{self, TierBatchBytes}; -use crate::placement::{PlacementError, Role}; pub use place_hybrid::{KvLayout, PlaceError}; @@ -134,41 +134,17 @@ pub const TIER_CHUNK: u64 = 16; /// (an NVFP4 down's 32-value blocks, a K- or i-quant down's 128-value form) and the host's rows /// and places. The plan reserves them on the tier card and the host, and the load checks its /// allocations against them. -#[must_use] -pub fn tier_batch(inputs: &PlanInputs, cols: u64) -> TierBatchBytes { +pub fn tier_batch(inputs: &PlanInputs, cols: u64) -> Result { let hp = &inputs.hp; - workstation::tier_leg_batch_bytes_of( + workstation::try_tier_leg_batch_bytes_of( hp.n_embd as u64, hp.experts.ff as u64, hp.experts.n_used as u64, (TIER_CHUNK, cols), - leg_downs(inputs), + accepts::leg_down_forms(Program::MiniMaxM3Body, &inputs.model), ) } -/// The forms the card leg's routed downs of `inputs` read: each routed layer's down stack whose -/// type the leg runs ([`planes_leg_down`]), by its kernel table row ([`kernels::down`]); the -/// 128-value form on a file with none. -fn leg_downs(inputs: &PlanInputs) -> DownForms { - let downs = (0..inputs.model.layers) - .map(|l| routed_stacks(l)[2].clone()) - .filter_map(|name| { - inputs - .model - .tensors - .iter() - .find(|t| t.role == Role::RoutedExperts && t.name == name) - }) - .filter(|t| planes_leg_down(t.ty)) - .filter_map(|t| kernels::down(t.ty)) - .fold(DownForms::NONE, |forms, (_, act)| forms.with(act)); - if downs == DownForms::NONE { - DownForms::of(DownAct::Q8Act) - } else { - downs - } -} - /// The slots of each half of the expert stream's ring a streaming plan keeps room for /// ([`stream_bytes`]) [derived: at the ring pass's route GEMMs the stream rule's crossover is about /// 18 columns of a walk, and a walk of the default group (two units of [`PROMPT_ROWS`] rows, four @@ -418,32 +394,18 @@ mod tests { )); } - /// A machine that names an expert tier card (plan (b′)) is planned: the body serves one. The - /// card keeps the prompt batch's reserve on either machine and, on the tiered one, the tier - /// places' copies beside it ([`tier_places_bytes`]: 3 × 4 B × 256 rows × 2 slots). The tier's batch bytes are the common leg's: runs of - /// [`TIER_CHUNK`] tokens over blocks of the unit's `cols` rows, in the forms the leg's routed - /// downs read: the header's F32 downs run on no leg, so the 128-value form - /// ([`workstation::tier_leg_batch_bytes`]). The plan reaches the placement, which refuses the - /// tier idle: the header's experts all fit the stage card. Mutant: the tier reserve left out - /// of the description (the plan stops at [`PlaceError::Tier`]), the forms of a file with no - /// leg down left empty (the scratch holds no slot columns), or the run's tokens and the - /// block's rows swapped. + /// A tiered machine retains the stage's prompt reserve and the tier places beside it. + /// The fixture's experts fit on the stage, so placement refuses the idle tier rather + /// than rejecting the description as a body without tier support. #[test] fn a_tier_card_is_planned_with_the_leg_batch() { const COLS: u64 = 3 * TIER_CHUNK; let inputs = described("mm3-place-tier"); - let hp = &inputs.hp; - assert_eq!( - tier_batch(&inputs, COLS), - workstation::tier_leg_batch_bytes( - hp.n_embd as u64, - hp.experts.ff as u64, - hp.experts.n_used as u64, - TIER_CHUNK, - COLS, - ) + let tiered = workstation::plan_bp( + LAYERS, + None, + tier_batch(&inputs, COLS).expect("valid tier geometry"), ); - let tiered = workstation::plan_bp(LAYERS, None, tier_batch(&inputs, COLS)); let plain = workstation::plan_gate(LAYERS); let places = 3 * 4 * 256 * 2; assert_eq!(inputs.tier_reserve, Some(places)); diff --git a/crates/model/src/arch/musespark/place.rs b/crates/model/src/arch/musespark/place.rs index 01572dc42..f0900aced 100644 --- a/crates/model/src/arch/musespark/place.rs +++ b/crates/model/src/arch/musespark/place.rs @@ -76,8 +76,7 @@ pub const fn pass_rows_max(n_used: usize) -> usize { /// placed pass's rows ([`pass_rows_max`]): the experts run in the latent, so /// the tier's rows and its staging are the latent's width, through the card /// leg in chunks of [`TIER_CHUNK`]. -#[must_use] -pub fn tier_batch(hp: &Hparams) -> TierBatchBytes { +pub fn tier_batch(hp: &Hparams) -> Result { let ff = hp.expert_width(); workstation::tier_leg_batch_bytes( hp.latent as u64, @@ -640,6 +639,7 @@ mod tests { head: true, token_embedding: false, reserve_only: false, + tier_leg: None, reserves: Vec::new(), }], tiers: Vec::new(), diff --git a/crates/model/src/arch/qwen35moe/place.rs b/crates/model/src/arch/qwen35moe/place.rs index 0e5fd350a..c85d9bbb0 100644 --- a/crates/model/src/arch/qwen35moe/place.rs +++ b/crates/model/src/arch/qwen35moe/place.rs @@ -52,7 +52,7 @@ use super::hparams::Kind; use super::{mtp, names}; use crate::arch::{architecture_is, coverage}; use crate::placement::accepts::{ - Program, leg_card_format, program_of, qwen38_down, qwen38_gate_up, routed_rule, + Program, leg_card_format, leg_down_forms, program_of, qwen38_down, qwen38_gate_up, routed_rule, }; use crate::placement::ctx::{CtxLimit, PastLimit, check_slot_ctx}; use crate::placement::ctx_ask::PlanRefusal; @@ -1498,17 +1498,19 @@ pub const LEG_TIER_CHUNK: u64 = 8; /// card's staging, one chunk of the leg's scratch and the host's rows, for /// blocks of the host union's columns. [`tier_batch`] is the qwen4exp plan's /// figure. -#[must_use] -pub fn leg_tier_batch(spec: &ModelSpec) -> TierBatchBytes { - let (expert_ff, n_used) = spec - .routed() - .map_or((0, 0), |m| (u64::from(m.expert_ff), u64::from(m.top_k))); - workstation::tier_leg_batch_bytes( +pub fn leg_tier_batch(inputs: &PlanInputs) -> Result { + let spec = &inputs.spec; + let moe = spec.routed().ok_or_else(|| PlacementError::Metadata { + key: "Qwen3.6 expert tier".into(), + detail: "a dense model has no routed experts for an expert tier".into(), + })?; + let (expert_ff, n_used) = (u64::from(moe.expert_ff), u64::from(moe.top_k)); + workstation::try_tier_leg_batch_bytes_of( u64::from(spec.hidden), expert_ff, n_used, - LEG_TIER_CHUNK, - crate::moe::UNION_MAX_COLS as u64, + (LEG_TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64), + leg_down_forms(Program::Qwen35Body, &inputs.model), ) } @@ -1543,6 +1545,7 @@ const fn tier_batch_of(n_embd: u64, n_used: u64, ubatch: u64) -> TierBatchBytes staging: tier_batch_staging_bytes(n_embd, n_used, cols), scratch: tier_route_scratch_bytes(cols), host: tier_batch_host_bytes(n_embd, n_used, cols), + leg: None, } } @@ -2508,6 +2511,7 @@ fn draft_machine() -> Machine { Machine { cards: vec![Card { name: "mtp draft".to_string(), + tier_leg: None, device: None, usable_bytes: u64::MAX, context_bytes: 0, @@ -3092,6 +3096,7 @@ mod tests { Machine { cards: vec![Card { name: "card".to_string(), + tier_leg: None, device: None, usable_bytes: usable, context_bytes: 0, @@ -3920,6 +3925,7 @@ mod tests { Machine { cards: vec![Card { name: "card".to_string(), + tier_leg: None, device: None, usable_bytes: 1 << 40, context_bytes: 0, @@ -4834,6 +4840,7 @@ mod tests { staging: 496_730_112, scratch: 348_980_248, host: 839_188_480, + leg: None, } ); assert_eq!( @@ -4842,6 +4849,7 @@ mod tests { staging: 62_091_264, scratch: 87_233_816, host: 104_898_560, + leg: None, } ); } diff --git a/crates/model/src/arch/qwen3moe/place.rs b/crates/model/src/arch/qwen3moe/place.rs index 3c71a02d2..7d2aa664a 100644 --- a/crates/model/src/arch/qwen3moe/place.rs +++ b/crates/model/src/arch/qwen3moe/place.rs @@ -19,7 +19,7 @@ use gguf::Split; use super::hparams::Hparams; use super::roles; -use crate::placement::accepts::{Program, routed_rule}; +use crate::placement::accepts::{Program, leg_down_forms, routed_rule}; use crate::placement::ctx_ask::PlanRefusal; use crate::placement::workstation::{self, TierBatchBytes}; use crate::placement::{ @@ -45,14 +45,14 @@ pub const TIER_CHUNK: u64 = 8; /// The expert tier's prompt-batch bytes ([`TierBatchBytes`]) for blocks of /// the host union's columns ([`crate::moe::UNION_MAX_COLS`]): the tier card's /// staging, one chunk of the card leg's scratch and the host's rows. -#[must_use] -pub fn tier_batch(hp: &Hparams) -> TierBatchBytes { - workstation::tier_leg_batch_bytes( +pub fn tier_batch(inputs: &PlanInputs) -> Result { + let hp = &inputs.hp; + workstation::try_tier_leg_batch_bytes_of( hp.n_embd as u64, hp.experts.ff as u64, hp.experts.n_used as u64, - TIER_CHUNK, - crate::moe::UNION_MAX_COLS as u64, + (TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64), + leg_down_forms(PROGRAM, &inputs.model), ) } diff --git a/crates/model/src/arch/solar2/place.rs b/crates/model/src/arch/solar2/place.rs index 95f0e5f15..849e38fba 100644 --- a/crates/model/src/arch/solar2/place.rs +++ b/crates/model/src/arch/solar2/place.rs @@ -14,9 +14,10 @@ use super::hparams::Hparams; use super::{roles, spec}; use crate::arch::chat_of; use crate::arch::place_hybrid; +use crate::placement::accepts::{Program, leg_down_forms}; use crate::placement::kernels::q8_act_bytes; use crate::placement::workstation::TierBatchBytes; -use crate::placement::{PlacementError, workstation}; +use crate::placement::{ModelTensors, PlacementError, workstation}; pub use place_hybrid::{GEMM_TILE_COLS, GEMM32_STEP, KvLayout, PlaceError}; use place_hybrid::{route_bytes, tier_places_bytes, wide_bytes}; @@ -40,18 +41,18 @@ pub const PARTIAL_ROWS: usize = 8; /// the stage card's card experts, which the body binds to its own constant. pub const TIER_CHUNK: u64 = 8; -/// The expert tier's prompt-batch bytes ([`TierBatchBytes`]) for a file of hyperparameters `hp`: +/// The expert tier's prompt-batch bytes ([`TierBatchBytes`]) for a described file: /// the tier card's staging, the leg's chunk scratch and the host's rows and places, for blocks of /// the host union's columns. The plan reserves them on the tier card and the host, and the load /// checks its allocations against them. -#[must_use] -pub fn tier_batch(hp: &Hparams) -> TierBatchBytes { - workstation::tier_leg_batch_bytes( +pub fn tier_batch(inputs: &PlanInputs) -> Result { + let hp = &inputs.hp; + workstation::try_tier_leg_batch_bytes_of( hp.n_embd as u64, hp.experts.ff as u64, hp.experts.n_used as u64, - TIER_CHUNK, - crate::moe::UNION_MAX_COLS as u64, + (TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64), + leg_down_forms(Program::Solar2Body, &inputs.model), ) } @@ -89,8 +90,12 @@ const fn acts_bytes(k: usize) -> usize { /// The wide forms are a bound: both forms at each width that allows them, where the load makes /// the ones its sites' types read. The units a prefill group past one adds are made out of the /// card's margin. -#[must_use] -pub fn prompt_reserve_bytes(hp: &Hparams, spec: &ModelSpec, rows: usize) -> u64 { +pub fn prompt_reserve_bytes( + hp: &Hparams, + spec: &ModelSpec, + model: &ModelTensors, + rows: usize, +) -> Result { let (embd, r) = (hp.n_embd, rows); let (mut heads, mut kv_heads, mut head) = (hp.n_head, 0, hp.head_dim); let (mut k_heads, mut v_heads, mut d) = (0, 0, 0); @@ -124,13 +129,21 @@ pub fn prompt_reserve_bytes(hp: &Hparams, spec: &ModelSpec, rows: usize) -> u64 let wide: usize = others.iter().map(|&k| wide_bytes(r, k)).sum::() + 2 * wide_bytes(r, embd) + wide_bytes(r, shared_ff); - let card_leg = workstation::tier_chunk_scratch_bytes( - embd as u64, - experts.ff as u64, - experts.n_used as u64, - r as u64, - ) as usize; - (4 * f32s + acts + wide + 2 * route_bytes(r) + card_leg) as u64 + let downs = leg_down_forms(Program::Solar2Body, model); + let card_leg = workstation::TierLegScratch { + n_embd: embd as u64, + ff: experts.ff as u64, + n_used: experts.n_used as u64, + chunk: r as u64, + downs, + } + .bytes(downs)?; + ((4 * f32s + acts + wide + 2 * route_bytes(r)) as u64) + .checked_add(card_leg) + .ok_or_else(|| PlacementError::Metadata { + key: "solar2 prompt reserve".into(), + detail: "bytes overflow u64".into(), + }) } impl PlanInputs { @@ -147,7 +160,7 @@ impl PlanInputs { let model = roles::classify(split, &hp)?; // No parser is declared: see `spec`. let spec = spec::spec_of(&hp, chat_of(split, None, None)?)?; - let reserve = prompt_reserve_bytes(&hp, &spec, PROMPT_ROWS); + let reserve = prompt_reserve_bytes(&hp, &spec, &model, PROMPT_ROWS)?; let tier = tier_places_bytes(PROMPT_ROWS, hp.experts.n_used); Ok(PlanInputs::new(ENGINE, hp, model, spec)? .with_prompt_reserve(reserve) @@ -163,8 +176,8 @@ mod tests { use super::super::hparams::tests::{LAYERS, keys, tensors}; use super::{PROMPT_ROWS, PlaceError, PlanInputs, prompt_reserve_bytes, tier_batch}; - use crate::arch::synthetic::{V, header_shaped}; - use crate::placement::{Device, PlacementError, PlanLevers, Violation, workstation}; + use crate::arch::synthetic::{Ty, V, header_shaped, header_typed}; + use crate::placement::{Device, PlacementError, PlanLevers, workstation}; /// The small header's file, described; each test names its own file (the tests run in /// threads of one process). @@ -247,7 +260,7 @@ mod tests { #[test] fn a_tier_card_is_planned_with_the_batch_places_beside_the_reserve() { let inputs = described("so2-place-tier"); - let tiered = workstation::plan_bp(LAYERS, None, tier_batch(&inputs.hp)); + let tiered = workstation::plan_bp(LAYERS, None, tier_batch(&inputs).expect("tier bytes")); let plain = workstation::plan_gate(LAYERS); let places = 12 * 512 * inputs.hp.experts.n_used as u64; assert_eq!(inputs.tier_reserve, Some(places)); @@ -305,51 +318,118 @@ mod tests { /// expert. Mutant: the reserve left out of the plan's call (the card plans, and its load stops /// at the allocation). #[test] - fn a_card_short_of_the_prompt_reserve_is_refused_by_name() { - let inputs = described("so2-place-reserve"); - let reserve = prompt_reserve_bytes(&inputs.hp, &inputs.spec, PROMPT_ROWS); - assert!(reserve > 0, "the synthetic header's batch has bytes"); - assert_eq!(inputs.prompt_reserve, reserve, "describe keeps the figure"); + fn a_host_only_mixture_keeps_the_exact_prompt_reserve_boundary() { + let mut kv = keys(); + for (key, value) in &mut kv { + if matches!(*key, "embedding_length" | "expert_feed_forward_length") { + *value = V::U32(256); + } + } + let mut ts = tensors(); + for (name, dims) in &mut ts { + let tail = name.rsplit('.').nth(1).unwrap_or(""); + if matches!( + tail, + "ffn_gate_exps" + | "ffn_up_exps" + | "ffn_down_exps" + | "ffn_gate_shexp" + | "ffn_up_shexp" + | "ffn_down_shexp" + ) { + dims[0] = 256; + dims[1] = 256; + } else if tail == "attn_output" { + dims[1] = 256; + } else if matches!( + tail, + "token_embd" + | "output" + | "output_norm" + | "attn_norm" + | "ffn_norm" + | "attn_q" + | "attn_k" + | "attn_v" + | "attn_gate" + | "ssm_f_a" + | "ssm_g_a" + | "ssm_beta" + | "ffn_gate_inp" + ) { + dims[0] = 256; + } + } + let global = [ + ("tokenizer.ggml.pre", V::Str("solar-open")), + ("tokenizer.chat_template", V::Str("{{ messages }}")), + ]; + let ts: Vec<_> = ts + .into_iter() + .map(|(name, dims)| { + let ty = if name == "blk.0.ffn_down_exps.weight" { + Ty::Q8_0 + } else if matches!( + name.rsplit('.').nth(1), + Some("ffn_gate_exps" | "ffn_up_exps" | "ffn_down_exps") + ) { + Ty::Q4K + } else { + Ty::F32 + }; + (name, dims, ty) + }) + .collect(); + let path = header_typed("so2-place-mixed-reserve", "solar-open2", &kv, &global, &ts); + let split = Split::open(&path).expect("the mixed fixture opens"); + let inputs = PlanInputs::describe(&split).expect("valid routed geometry"); + std::fs::remove_file(path).expect("remove the header fixture"); + let reserve = prompt_reserve_bytes(&inputs.hp, &inputs.spec, &inputs.model, PROMPT_ROWS) + .expect("prompt reserve"); + let ample = workstation::plan_gate(LAYERS); + let placed = inputs + .plan(&le, 64, &PlanLevers::default()) + .expect("the mixed plan"); + assert_eq!( + placed.n_l[0], 0, + "Q8_0 stays on the host under Solar's rule" + ); + assert!( + placed.n_l[1..].iter().all(|&n| n > 0), + "other layers remain card candidates" + ); let levers = PlanLevers::default(); let mut machine = workstation::plan_gate(LAYERS); - machine.cards[0].usable_bytes = machine.cards[0].margin_bytes; - let short = loop { - match inputs.plan(&machine, 64, &levers) { - Err(PlaceError::Broken(v)) => { - let over = v - .iter() - .find_map(|v| match v { - Violation::CardOver { total, limit, .. } => Some(total - limit), - _ => None, - }) - .unwrap_or_else(|| panic!("{v:?}, no card over its limit")); - machine.cards[0].usable_bytes += over; - } - Err(PlaceError::PromptOver { - reserve: got, - left, - short, - .. - }) => { - assert_eq!((got, short), (reserve, reserve - left)); - break short; - } - other => panic!("{other:?}, not the reserve's refusal"), - } - }; - machine.cards[0].usable_bytes += short - 1; - assert!(matches!( - inputs.plan(&machine, 64, &levers), - Err(PlaceError::PromptOver { short: 1, .. }) - )); + // Rounded dense padding may hold experts for no extra card bytes. + // Byte granules isolate the reserve's exact no-expert-room boundary. + machine.cards[0].granule_bytes = std::num::NonZeroU64::MIN; + let host_only = crate::placement::plan_host_routed( + &inputs.model, + &machine, + 64, + &inputs.seq_terms().slots_of(1), + &levers, + ) + .expect("the host-only reference plans"); + let dense = crate::arch::place_hybrid::card_terms(&host_only.cards[0]); + machine.cards[0].usable_bytes = machine.cards[0].margin_bytes + dense + reserve - 1; + match inputs.plan(&machine, 64, &levers) { + Err(PlaceError::PromptOver { + reserve: got, + left, + short, + .. + }) => assert_eq!((got, left, short), (reserve, reserve - 1, 1)), + other => panic!("{other:?}, not the reserve's one-byte refusal"), + } machine.cards[0].usable_bytes += 1; let plan = inputs .plan(&machine, 64, &levers) .expect("the card holds the terms and the reserve"); - assert_eq!( - plan.n_l.iter().sum::(), - 0, - "no room is left for an expert" + assert!(plan.violations().is_empty()); + assert!( + plan.n_l.iter().all(|&n| n == 0), + "the exact dense-and-prompt budget leaves no expert room" ); } } diff --git a/crates/model/tests/mimo2_meta.rs b/crates/model/tests/mimo2_meta.rs index 64ecda04a..a48c53896 100644 --- a/crates/model/tests/mimo2_meta.rs +++ b/crates/model/tests/mimo2_meta.rs @@ -401,7 +401,7 @@ fn hw_mimo2_plan() { let tiered = workstation::plan_bp( layers, None, - model::arch::mimo2::place::tier_batch(&inputs.hp, 512), + model::arch::mimo2::place::tier_batch(&inputs.hp, 512).expect("valid tier geometry"), ); let bp = inputs .plan(&tiered, ctx, &PlanLevers::default()) diff --git a/crates/model/tests/musespark_meta.rs b/crates/model/tests/musespark_meta.rs index d3cabdd9c..079fc4c19 100644 --- a/crates/model/tests/musespark_meta.rs +++ b/crates/model/tests/musespark_meta.rs @@ -368,7 +368,7 @@ fn hw_musespark_plan() { workstation::A6000, &[workstation::RTX_3090], None, - place::tier_batch(&hp), + place::tier_batch(&hp).expect("valid tier geometry"), ) .unwrap_or_else(|e| panic!("the tiered machine: {e}")); place::stage_placed(&mut m).unwrap_or_else(|e| panic!("the tiered machine's stage: {e}")); diff --git a/crates/model/tests/placement.rs b/crates/model/tests/placement.rs index c62b0921c..394803f57 100644 --- a/crates/model/tests/placement.rs +++ b/crates/model/tests/placement.rs @@ -1371,6 +1371,7 @@ fn layered_model(layers: usize) -> ModelTensors { fn byte_card(name: &str, usable: u64, layers: Range) -> Card { Card { name: name.to_string(), + tier_leg: None, device: None, usable_bytes: usable, context_bytes: 0, diff --git a/crates/model/tests/qwen3_bp_plan.rs b/crates/model/tests/qwen3_bp_plan.rs index ff86bc0f0..da3cf6dc0 100644 --- a/crates/model/tests/qwen3_bp_plan.rs +++ b/crates/model/tests/qwen3_bp_plan.rs @@ -522,7 +522,7 @@ fn qwen36_inputs() -> q35::PlanInputs { fn qwen36_tier(stage: CardSpec, tier: CardSpec) -> Result<(u64, u64, u64), Place35Error> { let inputs = qwen36_inputs(); - let machine = tiered(Q36_LAYERS, stage, tier, q35::leg_tier_batch(&inputs.spec)); + let machine = tiered(Q36_LAYERS, stage, tier, q35::leg_tier_batch(&inputs)?); let plan = inputs.plan_rule( &machine, CTX, @@ -537,7 +537,12 @@ fn qwen36_tier(stage: CardSpec, tier: CardSpec) -> Result<(u64, u64, u64), Place #[test] fn qwen3_30b_q8_bp_tier_experts() { let inputs = qwen3_inputs(); - let machine = tiered(Q3_LAYERS, A6000, RTX_3090, q3::tier_batch(&inputs.hp)); + let machine = tiered( + Q3_LAYERS, + A6000, + RTX_3090, + q3::tier_batch(&inputs).expect("valid tier geometry"), + ); let plan = inputs .plan(&machine, CTX, &PlanLevers::default()) .expect("the A6000 stage plans a tier"); diff --git a/crates/model/tests/qwen4exp_mtp_reserve.rs b/crates/model/tests/qwen4exp_mtp_reserve.rs index d4e0558a7..ed3e831ea 100644 --- a/crates/model/tests/qwen4exp_mtp_reserve.rs +++ b/crates/model/tests/qwen4exp_mtp_reserve.rs @@ -328,6 +328,7 @@ fn machine(usable: u64) -> Machine { Machine { cards: vec![Card { name: "card".to_string(), + tier_leg: None, device: None, usable_bytes: usable, context_bytes: CONTEXT, diff --git a/crates/model/tests/solar_open2_meta.rs b/crates/model/tests/solar_open2_meta.rs index 29e99622f..05f581acf 100644 --- a/crates/model/tests/solar_open2_meta.rs +++ b/crates/model/tests/solar_open2_meta.rs @@ -400,6 +400,7 @@ fn hw_solar_open2_plan_a() { staging: 0, scratch: 0, host: 0, + leg: None, }; let bp = workstation::plan_bp(layers, None, tier_batch); let tiered = inputs diff --git a/crates/placement/src/accepts.rs b/crates/placement/src/accepts.rs index 5b4273f67..617421833 100644 --- a/crates/placement/src/accepts.rs +++ b/crates/placement/src/accepts.rs @@ -531,6 +531,49 @@ pub fn leg_runs(p: Program, [gate, up, down]: [GgmlType; 3]) -> Option { }) } +/// The union of down activation forms of the complete routed layers that +/// `p`'s placement rule admits, resolved from the file's descriptor at load. +#[must_use] +pub fn leg_down_forms(p: Program, model: &crate::placement::ModelTensors) -> kernels::DownForms { + use crate::placement::{ExpertPart, Role}; + + let RoutedRule::Leg { + gate_up, + down, + pair, + } = routed_rule(p) + else { + return kernels::DownForms::NONE; + }; + let mut forms = kernels::DownForms::NONE; + for l in 0..model.layers { + let mut parts = [None; 3]; + for t in model + .tensors + .iter() + .filter(|t| t.role == Role::RoutedExperts && t.layer == Some(l)) + { + let part = match ExpertPart::of_name(&t.name) { + Some(ExpertPart::Gate) => 0, + Some(ExpertPart::Up) => 1, + Some(ExpertPart::Down) => 2, + None => continue, + }; + parts[part] = Some(t.ty); + } + if let [Some(g), Some(u), Some(d)] = parts + && g == u + && gate_up(g).is_some() + && down(d).is_some() + && pair.is_none_or(|pair| pair(g, d, false)) + && let Some((_, act)) = kernels::down(d) + { + forms = forms.with(act); + } + } + forms +} + /// Program `p`'s plan rule for its routed stacks ([`RoutedRule`]): on a card /// leg, each part by the arm its load reads ([`leg_runs`]), so a layer the /// plan gives card experts is one the load runs. The mimo2 and solar2 diff --git a/crates/placement/src/kernels.rs b/crates/placement/src/kernels.rs index 6dfe81f83..c9b912b2e 100644 --- a/crates/placement/src/kernels.rs +++ b/crates/placement/src/kernels.rs @@ -104,12 +104,30 @@ pub const fn embed(ty: GgmlType) -> Option { /// it holds; a form whose last super-block is padded passes the padded width. #[must_use] pub const fn q8_act_bytes(cols: u64, k: u64) -> u64 { - let n_sb = k / 256; - 8 * cols * 64 * n_sb.div_ceil(2) - + 4 * cols * 256 * n_sb.div_ceil(4) - + 4 * cols * 128 * n_sb.div_ceil(2) - + 4 * cols * 8 * n_sb - + 4 * cols * 2 * n_sb + match try_q8_act_bytes(cols, k) { + Some(bytes) => bytes, + None => panic!("q8_act_bytes: activation bytes overflow u64"), + } +} + +/// [`q8_act_bytes`] with overflow returned to the placement boundary. +#[must_use] +pub(crate) const fn try_q8_act_bytes(cols: u64, k: u64) -> Option { + let n_sb = (k / 256) as u128; + // A u64 width's per-column count fits u128; its column product may not. + let per_col = 512 * n_sb.div_ceil(2) + + 1024 * n_sb.div_ceil(4) + + 512 * n_sb.div_ceil(2) + + 32 * n_sb + + 8 * n_sb; + let Some(bytes) = per_col.checked_mul(cols as u128) else { + return None; + }; + if bytes > u64::MAX as u128 { + None + } else { + Some(bytes as u64) + } } /// The card bytes of the 32-value q8_1 activation of `cols` columns of `k` @@ -118,8 +136,25 @@ pub const fn q8_act_bytes(cols: u64, k: u64) -> u64 { /// and scales, four bytes each, `k_blocks = k / 32`. #[must_use] pub const fn q8_blocks32_bytes(cols: u64, k: u64) -> u64 { - let k_blocks = k / 32; - cols * 4 * (256 * k_blocks.div_ceil(32) + 2 * k_blocks) + match try_q8_blocks32_bytes(cols, k) { + Some(bytes) => bytes, + None => panic!("q8_blocks32_bytes: activation bytes overflow u64"), + } +} + +/// [`q8_blocks32_bytes`] with overflow returned to the placement boundary. +#[must_use] +const fn try_q8_blocks32_bytes(cols: u64, k: u64) -> Option { + let k_blocks = (k / 32) as u128; + let per_col = 4 * (256 * k_blocks.div_ceil(32) + 2 * k_blocks); + let Some(bytes) = per_col.checked_mul(cols as u128) else { + return None; + }; + if bytes > u64::MAX as u128 { + None + } else { + Some(bytes as u64) + } } /// The down activation forms ([`DownAct`]) a card leg's buffers hold: the @@ -178,30 +213,68 @@ impl DownForms { /// them read it. #[must_use] pub const fn leg_bytes(cap: u64, slots_per_col: u64, n: u64, ff: u64, downs: DownForms) -> u64 { - let slots = cap * slots_per_col; + match try_leg_bytes(cap, slots_per_col, n, ff, downs) { + Some(bytes) => bytes, + None => panic!("leg_bytes: scratch bytes overflow u64"), + } +} + +/// [`leg_bytes`] with overflow returned to the placement boundary. This +/// counts the same whole blocks and padded planes as the infallible sizing. +#[must_use] +pub(crate) const fn try_leg_bytes( + cap: u64, + slots_per_col: u64, + n: u64, + ff: u64, + downs: DownForms, +) -> Option { + let Some(slots) = cap.checked_mul(slots_per_col) else { + return None; + }; + let Some(row_bytes) = slots.checked_mul(4) else { + return None; + }; + let Some(h) = row_bytes.checked_mul(ff) else { + return None; + }; + let Some(down) = row_bytes.checked_mul(n) else { + return None; + }; + let Some(input) = try_q8_act_bytes(cap, n) else { + return None; + }; let q8 = if downs.q8_act { - q8_act_bytes(slots, ff) + let Some(bytes) = try_q8_act_bytes(slots, ff) else { + return None; + }; + bytes } else { 0 }; let b32 = if downs.blocks32 { - q8_blocks32_bytes(slots, ff) + let Some(bytes) = try_q8_blocks32_bytes(slots, ff) else { + return None; + }; + bytes } else { 0 }; - q8_act_bytes(cap, n) + 4 * slots * ff + q8 + b32 + 4 * slots * n + let bytes = input as u128 + h as u128 + q8 as u128 + b32 as u128 + down as u128; + if bytes > u64::MAX as u128 { + None + } else { + Some(bytes as u64) + } } /// Whether the common card leg (`crates/gpu/src/card_leg.rs`) launches the /// gate·up entry of a routed gate or up stack of type `ty` in the file's /// blocks ([`crate::placement::CardFormat::KQuant`]): the table's [`gate_up`] /// row, for the entries whose launchers read a `Q8Act` of more columns than a -/// run uses. The Q8_0 row has a launcher of that shape and an arm in the leg -/// (`CardLeg::launch`), which this set leaves out: the families that plan by -/// it (GLM-5.3, MiMo, Solar-Open2, MiniMax-M3, Raon) hold `Q8Act` down buffers -/// only, so a Q8_0 layer they planned on the card would fail at its first -/// launch where it runs on the host today; a program whose buffers follow the -/// down's form reads the arm through its own row ([`crate::accepts::prompt_leg_gate_up`]). +/// run uses. The Q8_0 and Q6_K rows have launchers of that shape and arms in the +/// leg, but only programs whose selected gate·up policy admits them read those +/// arms ([`crate::accepts::prompt_leg_gate_up`]). /// A stack in its planes is [`card_planes_runs`]'. #[must_use] pub const fn card_gate_up_runs(ty: GgmlType) -> bool { @@ -335,6 +408,54 @@ mod tests { use super::*; use crate::placement::CardFormat; + /// Plane-padding transitions retain their byte counts, and both sizing + /// contracts share the last representable column and aggregate boundary. + #[test] + fn activation_and_leg_bytes_keep_padding_and_overflow_boundaries() { + for (k, bytes) in [(1024, 3232), (1280, 5320)] { + assert_eq!(try_q8_act_bytes(1, k), Some(bytes)); + assert_eq!(q8_act_bytes(1, k), bytes); + } + for (k, bytes) in [(1024, 1280), (1056, 2312)] { + assert_eq!(try_q8_blocks32_bytes(1, k), Some(bytes)); + assert_eq!(q8_blocks32_bytes(1, k), bytes); + } + let q8 = DownForms::of(DownAct::Q8Act); + let b32 = DownForms::of(DownAct::Q8Blocks32); + for (forms, bytes) in [ + (DownForms::NONE, 4136), + (q8, 6224), + (b32, 5224), + (q8.with(DownAct::Q8Blocks32), 7312), + ] { + assert_eq!(try_leg_bytes(1, 1, 256, 256, forms), Some(bytes)); + assert_eq!(leg_bytes(1, 1, 256, 256, forms), bytes); + } + assert_eq!(try_leg_bytes(1, 1, 64, 24, DownForms::NONE), Some(352)); + let act_cols = u64::MAX / 2088; + assert_eq!(try_q8_act_bytes(act_cols, 256), Some(act_cols * 2088)); + assert_eq!(q8_act_bytes(act_cols, 256), act_cols * 2088); + assert_eq!(try_q8_act_bytes(act_cols + 1, 256), None); + let block_cols = u64::MAX / 1088; + assert_eq!( + try_q8_blocks32_bytes(block_cols, 256), + Some(block_cols * 1088) + ); + assert_eq!(q8_blocks32_bytes(block_cols, 256), block_cols * 1088); + assert_eq!(try_q8_blocks32_bytes(block_cols + 1, 256), None); + let cap = u64::MAX / 6224; + assert_eq!(try_leg_bytes(cap, 1, 256, 256, q8), Some(cap * 6224)); + assert_eq!(leg_bytes(cap, 1, 256, 256, q8), cap * 6224); + assert_eq!(try_leg_bytes(cap + 1, 1, 256, 256, q8), None); + assert_eq!(try_leg_bytes(1 << 61, 8, 256, 256, q8), None); + } + + #[test] + #[should_panic(expected = "leg_bytes: scratch bytes overflow u64")] + fn infallible_leg_sizing_refuses_overflow() { + let _ = leg_bytes(1 << 61, 8, 256, 256, DownForms::of(DownAct::Q8Act)); + } + /// Every `GgmlType` variant in declaration order, then one id the enum /// does not name. const ALL: [GgmlType; 42] = [ diff --git a/crates/placement/src/placement.rs b/crates/placement/src/placement.rs index 0c8a6bdca..f470d2617 100644 --- a/crates/placement/src/placement.rs +++ b/crates/placement/src/placement.rs @@ -250,6 +250,8 @@ pub struct Card { /// draft model's resident bytes, say): taken from the budget with the /// context and the scratch ([`Card::set_aside_bytes`]). pub reserves: Vec<(String, u64)>, + /// A tier leg's scratch, jointly selected with its resident experts. + pub tier_leg: Option, /// A tier card with an expert cap of 0: it holds its reserves and no /// expert, and serves no tier. The plan refuses one with no reserve /// ([`PlacementError::IdleTier`]); a body hangs a tier only under the @@ -276,10 +278,49 @@ impl Card { .try_fold(0u64, |sum, (_, bytes)| sum.checked_add(*bytes)) } - /// What the card sets aside before any tensor or cache: context, - /// scratch and reserves. The one owner of that sum: the expert rule's - /// budget, the budget floor, the headroom and [`Plan::violations`] all - /// take it from here. The margin is not in it. `None` past u64 bytes. + /// The tier leg's bound and selected scratch; absent on fixed-reserve paths. + fn selected_scratch( + &self, + model: &ModelTensors, + counts: &[u64], + ) -> Result, PlacementError> { + let Some(leg) = self.tier_leg else { + return Ok(None); + }; + Ok(Some(( + leg.bytes(leg.downs)?, + selected_leg_scratch( + leg, + counts, + routed_down_acts(model).map(|(l, act)| (l, crate::kernels::DownForms::of(act))), + )?, + ))) + } + + fn selected_reserve_bytes( + &self, + model: &ModelTensors, + counts: &[u64], + ) -> Result { + let fixed = self.reserve_bytes().ok_or_else(|| self.bytes_past_u64())?; + match self.selected_scratch(model, counts)? { + Some((bound, selected)) => fixed + .checked_sub(bound) + .and_then(|n| n.checked_add(selected)) + .ok_or_else(|| self.bytes_past_u64()), + None => Ok(fixed), + } + } + + fn floor_with_reserves(&self, dense: u64, kv: u64, past: u64, reserves: u64) -> Option { + [kv, self.context_bytes, self.scratch_bytes, reserves, past] + .into_iter() + .try_fold(dense, u64::checked_add) + } + + /// The machine's context, scratch and reserve bound before uploads. + /// Placement replaces a tier leg's bound with its candidate's actual + /// scratch. The margin is not in it. `None` past u64 bytes. #[must_use] pub fn set_aside_bytes(&self) -> Option { self.context_bytes @@ -287,18 +328,13 @@ impl Card { .checked_add(self.reserve_bytes()?) } - /// The card's floor under `dense` bytes of uploads and `kv` bytes of - /// cache: those, its set-aside, and `past` — what the load keeps beside - /// them: a plan's margin and what its caller holds beside the plan - /// (`plan_floor` sums those), or a whole load's own arena and reserve - /// ([`WholeLoad::past_bytes`]). The one owner of that sum: the budget - /// floor, the free floor and the whole fit ([`whole_need`]) all take it - /// from here. `None` past u64 bytes. + /// The fixed/bounded floor under `dense` upload bytes, `kv` cache bytes + /// and `past` bytes held beside them ([`WholeLoad::past_bytes`]). + /// Selected plans use the same sum with their actual reserves. + /// `None` past u64 bytes. #[must_use] pub fn floor_bytes(&self, dense: u64, kv: u64, past: u64) -> Option { - [kv, self.set_aside_bytes()?, past] - .into_iter() - .try_fold(dense, u64::checked_add) + self.floor_with_reserves(dense, kv, past, self.reserve_bytes()?) } fn bytes_past_u64(&self) -> PlacementError { @@ -330,6 +366,41 @@ impl Card { } } +/// Routed down forms with their layers, before a placement selects any experts. +fn routed_down_acts( + model: &ModelTensors, +) -> impl Iterator + '_ { + model.tensors.iter().filter_map(|t| { + if t.role != Role::RoutedExperts || ExpertPart::of_name(&t.name) != Some(ExpertPart::Down) { + return None; + } + Some((t.layer?, crate::kernels::down(t.ty)?.1)) + }) +} + +/// Price the actual selected union, from tensors or cached per-layer forms. +/// Empty counts and layers with no down keep the descriptor's empty-form scratch. +fn selected_leg_scratch( + leg: workstation::TierLegScratch, + counts: &[u64], + downs: impl IntoIterator, +) -> Result { + use crate::kernels::{DownAct, DownForms}; + + let forms = downs + .into_iter() + .filter(|(l, _)| counts.get(*l).is_some_and(|&n| n > 0)) + .fold(DownForms::NONE, |mut forms, (_, down)| { + for act in [DownAct::Q8Act, DownAct::Q8Blocks32] { + if down.holds(act) { + forms = forms.with(act); + } + } + forms + }); + leg.bytes(forms) +} + /// The host tier: its usable bytes and what is set aside before any tensor. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Host { @@ -1011,7 +1082,8 @@ pub struct CardTotals { pub stage_rows_bytes: u64, pub scratch_bytes: u64, pub context_bytes: u64, - /// The card's reserves ([`Card::reserves`]). + /// The selected reserves; a tier leg's descriptor bound is replaced by + /// its actual resident layers' scratch. pub reserve_bytes: u64, /// usable − dense − experts − rounding − KV − stage rows − scratch − /// context − reserves, usable capped by the plan's card budget; the @@ -2915,7 +2987,8 @@ fn resolve_shares( let uploads = card_uploads(model, c, rows, &own, routed)?; let dense = Footprint::new(card.granule_bytes, &uploads).at(&zeros)?; let kv = kv_card.saturating_add(stage_rows); - let floor = plan_floor(card, dense, kv, reserve); + let reserves = card.selected_reserve_bytes(model, &zeros)?; + let floor = plan_floor(card, dense, kv, reserve, reserves); let cap = capped(card, None); let Some((floor, room)) = floor.and_then(|f| Some((f, cap.checked_sub(f)?))) else { return Err(refuse(format!( @@ -2927,7 +3000,7 @@ fn resolve_shares( kv, card.context_bytes, card.scratch_bytes, - card.reserve_bytes().unwrap_or(u64::MAX), + reserves, card.margin_bytes, ))); }; @@ -3065,6 +3138,36 @@ impl Fill<'_, '_> { .routed .unwrap_or(RoutedRule::PerType(CardFormat::of_routed)); let uploads = card_uploads(self.model, self.c, self.rows, self.eligible, format)?; + // Only tier legs prepare the per-layer forms. Candidates reuse these + // rather than scanning tensors, including when a layer has several downs. + let mut down_forms = if card.tier_leg.is_some() { + vec![crate::kernels::DownForms::NONE; self.model.layers] + } else { + Vec::new() + }; + if card.tier_leg.is_some() { + for (l, act) in routed_down_acts(self.model) { + if let Some(forms) = down_forms.get_mut(l) { + *forms = forms.with(act); + } + } + } + let scratch = |counts: &[u64]| { + card.tier_leg.map_or(Ok(0), |leg| { + selected_leg_scratch(leg, counts, down_forms.iter().copied().enumerate()) + }) + }; + let bound = card + .tier_leg + .map(|leg| leg.bytes(leg.downs)) + .transpose()? + .unwrap_or(0); + let selected = scratch(n_l)?; + let reserves = card + .reserve_bytes() + .and_then(|fixed| fixed.checked_sub(bound)) + .and_then(|fixed| fixed.checked_add(selected)) + .ok_or_else(|| card.bytes_past_u64())?; let footprint = Footprint::new(card.granule_bytes, &uploads); let budget = i128::from(capped(card, self.card_budget)) - i128::from(self.kv) @@ -3073,19 +3176,25 @@ impl Fill<'_, '_> { .ok_or_else(|| card.bytes_past_u64())?, ) - i128::from(card.margin_bytes) - - i128::from(self.reserve); + - i128::from(self.reserve) + + i128::from(bound); if let Some(b) = self.card_budget { let dense = footprint.at(n_l)?; - check_floor(card, b, dense, self.kv, self.reserve)?; + check_floor(card, b, dense, self.kv, self.reserve, reserves)?; } if let Some(free) = card.free_bytes.filter(|&f| f < card.usable_bytes) { let dense = footprint.at(n_l)?; - check_free_floor(card, free, dense, self.kv, self.reserve)?; + check_free_floor(card, free, dense, self.kv, self.reserve, reserves)?; } let caps: Vec = (0..n_l.len()) .map(|l| self.model.layer_experts(l)) .collect(); - spread(n_l, held, self.eligible, &caps, budget, |n| footprint.at(n)) + spread(n_l, held, self.eligible, &caps, budget, |n| { + footprint + .at(n)? + .checked_add(scratch(n)?) + .ok_or_else(|| card.bytes_past_u64()) + }) } } @@ -3093,14 +3202,13 @@ impl Fill<'_, '_> { /// the plan ([`Fill::reserve`]): [`Card::floor_bytes`] with the margin and /// those as its `past`. Both floor checks ([`check_floor`], /// [`check_free_floor`]) and the share split's floors ([`resolve_shares`]) -/// take it from here, so the floor they name is the least budget at which -/// the expert rule's own budget (usable less KV, set-aside, margin and -/// `beside`, in [`Fill::run`]) reaches the dense uploads. `None` past u64 -/// bytes. -fn plan_floor(card: &Card, dense: u64, kv: u64, beside: u64) -> Option { +/// take it from here, using each candidate's selected reserves, so the floor +/// is the least budget at which its uploads and scratch fit beside the +/// fixed terms. `None` past u64 bytes. +fn plan_floor(card: &Card, dense: u64, kv: u64, beside: u64, reserves: u64) -> Option { card.margin_bytes .checked_add(beside) - .and_then(|past| card.floor_bytes(dense, kv, past)) + .and_then(|past| card.floor_with_reserves(dense, kv, past, reserves)) } /// Refuse `card` under the card budget `budget` when its floor — the @@ -3115,8 +3223,9 @@ fn check_floor( dense: u64, kv: u64, beside: u64, + reserves: u64, ) -> Result<(), PlacementError> { - let floor = plan_floor(card, dense, kv, beside); + let floor = plan_floor(card, dense, kv, beside, reserves); match floor { Some(floor) if floor <= budget => Ok(()), _ => Err(PlacementError::CardBudgetFloor { @@ -3127,7 +3236,7 @@ fn check_floor( kv, context: card.context_bytes, scratch: card.scratch_bytes, - reserves: card.reserve_bytes().ok_or_else(|| card.bytes_past_u64())?, + reserves, margin: card.margin_bytes, beside, }), @@ -3146,9 +3255,10 @@ fn check_free_floor( dense: u64, kv: u64, beside: u64, + reserves: u64, ) -> Result<(), PlacementError> { let context = card.context_bytes - card.context_in_free(); - let floor = plan_floor(card, dense, kv, beside).map(|f| f - card.context_in_free()); + let floor = plan_floor(card, dense, kv, beside, reserves).map(|f| f - card.context_in_free()); match floor { Some(floor) if floor <= free => Ok(()), floor => Err(PlacementError::CardFreeFloor(Box::new(CardFreeFloor { @@ -3161,7 +3271,7 @@ fn check_free_floor( kv, context, scratch: card.scratch_bytes, - reserves: card.reserve_bytes().ok_or_else(|| card.bytes_past_u64())?, + reserves, margin: card.margin_bytes, beside, held: card @@ -4244,8 +4354,12 @@ fn totals<'a>( .map(|(c, (card, heap))| { let rounding = heap.taken - (card_dense[c] + card_experts[c]); let (kv, _, stage_rows) = kv_bytes[c]; + let counts = c + .checked_sub(machine.cards.len()) + .map_or(n_l.as_slice(), |t| tier_n_l[t].as_slice()); + let reserve_bytes = card.selected_reserve_bytes(model, counts)?; let used = card - .floor_bytes(heap.taken, kv, stage_rows) + .floor_with_reserves(heap.taken, kv, stage_rows, reserve_bytes) .ok_or_else(|| card.bytes_past_u64())?; if machine.unified().is_some() && machine.on_this_node(c) @@ -4265,9 +4379,7 @@ fn totals<'a>( stage_rows_bytes: stage_rows, scratch_bytes: card.scratch_bytes, context_bytes: card.context_bytes, - reserve_bytes: card - .reserve_bytes() - .expect("the card floor checked its reserves"), + reserve_bytes, headroom_bytes: i128::from(capped(card, card_budget::of(&card_budget, card))) - i128::from(used), }) @@ -4337,11 +4449,50 @@ pub struct PoolNeed { /// `card`'s total against usable − margin: its uploads' granules, its KV, /// its stage rows and its set-aside. fn card_total(card: &Card, totals: &CardTotals, heap: &Heap) -> u64 { - card.floor_bytes(heap.taken, totals.kv_bytes, totals.stage_rows_bytes) - .expect("the plan's card totals fit u64") + card.floor_with_reserves( + heap.taken, + totals.kv_bytes, + totals.stage_rows_bytes, + totals.reserve_bytes, + ) + .expect("the plan's card totals fit u64") } impl Plan<'_> { + /// One named reserve at its selected amount, not the machine's scratch + /// bound. Missing or duplicate rows and an invalid card return `None`; + /// invalid scratch descriptors propagate their named error. + pub fn card_reserve(&self, c: usize, name: &str) -> Result, PlacementError> { + let Some(card) = self.machine.card(c) else { + return Ok(None); + }; + let mut rows = card.reserves.iter().filter(|(n, _)| n == name); + let Some((_, bytes)) = rows.next() else { + return Ok(None); + }; + if rows.next().is_some() { + return Ok(None); + } + if name != workstation::TIER_BATCH_RESERVE || card.tier_leg.is_none() { + return Ok(Some(*bytes)); + } + let counts = match c.checked_sub(self.machine.cards.len()) { + None => self.n_l.as_slice(), + Some(t) => match self.tier_n_l.get(t) { + Some(counts) => counts.as_slice(), + None => return Ok(None), + }, + }; + match card.selected_scratch(self.model, counts)? { + Some((bound, selected)) => bytes + .checked_sub(bound) + .and_then(|bytes| bytes.checked_add(selected)) + .map(Some) + .ok_or_else(|| card.bytes_past_u64()), + None => Ok(Some(*bytes)), + } + } + fn reserve_beside(&mut self, reserve: u64) -> Result<(), PlacementError> { if self.machine.unified().is_some() { let here = (0..self.machine.all_cards().count()) @@ -4966,6 +5117,7 @@ mod tests { fn card(name: &str, layers: Range, head: bool, token_embedding: bool) -> Card { Card { name: name.to_string(), + tier_leg: None, device: None, usable_bytes: 0, context_bytes: 0, @@ -6315,6 +6467,176 @@ mod tests { assert_eq!(plan.n_l, vec![8]); } + /// Scratch enters each expert candidate's footprint monotonically. + /// A boundary that admits one down form cannot pay for an unused second + /// form, and adding the first expert of that form pays its exact bytes. + #[test] + fn a_tier_jointly_selects_expert_rows_and_down_forms() { + use crate::accepts::{Program, routed_rule}; + use crate::kernels::{DownAct, DownForms}; + use GgmlType::{Q5_K, Q8_0}; + use workstation::{TIER_BATCH_RESERVE, try_tier_leg_batch_bytes_of}; + + let both = DownForms::of(DownAct::Q8Act).with(DownAct::Q8Blocks32); + let mixed = try_tier_leg_batch_bytes_of(256, 256, 1, (2, 3), both).expect("tier bytes"); + let rule = routed_rule(Program::Qwen35Body); + for (downs, first, batch) in [ + ([Q5_K, Q8_0], DownAct::Q8Act, mixed), + ([Q8_0, Q5_K], DownAct::Q8Blocks32, mixed), + ( + [Q5_K; 2], + DownAct::Q8Act, + workstation::tier_leg_batch_bytes(256, 256, 1, 2, 3).expect("valid tier bytes"), + ), + ] { + let scratch = |forms| crate::kernels::leg_bytes(2, 1, 256, 256, forms); + let selected = + DownForms::of(first).with(crate::kernels::down(downs[1]).expect("second down").1); + let mut model = legged(&[[Q5_K, Q5_K, downs[0]], [Q5_K, Q5_K, downs[1]]]); + model.experts = 1; + model.experts_used = 1; + for t in model + .tensors + .iter_mut() + .filter(|t| t.role == Role::RoutedExperts) + { + t.dims[1] = 256; + t.dims[2] = 1; + t.file_bytes *= 8; + } + let expert = |l| { + model + .tensors + .iter() + .filter(|t| t.role == Role::RoutedExperts && t.layer == Some(l)) + .map(|t| t.file_bytes / model.experts) + .sum::() + }; + let one = batch.staging + scratch(DownForms::of(first)) + expert(0); + let two = batch.staging + scratch(selected) + expert(0) + expert(1); + for (room, counts, forms) in [ + (one - 1, [0, 0], DownForms::NONE), + (one, [1, 0], DownForms::of(first)), + (two - 1, [1, 0], DownForms::of(first)), + (two, [1, 1], selected), + ] { + let mut tier = bytes_card("tier", room, 0..0); + tier.reserves + .push((TIER_BATCH_RESERVE.into(), batch.card().unwrap())); + tier.tier_leg = batch.leg; + let machine = Machine { + cards: vec![bytes_card("stage", HEAD, 0..2)], + tiers: vec![tier], + nodes: vec![Node { + host: host(), + unified: None, + }], + }; + let plan = plan_routed(&model, &machine, 1, &NoKv, &PlanLevers::default(), rule) + .expect("the joint plan"); + assert_eq!(plan.n_l, [0, 0]); + assert_eq!(plan.tier_n_l[0], counts, "{downs:?}, {room}"); + let selected = batch.staging + scratch(forms); + assert_eq!( + plan.card_reserve(1, TIER_BATCH_RESERVE) + .expect("selected reserve"), + Some(selected) + ); + assert_eq!(plan.cards[1].reserve_bytes, selected); + let rows = expert(0) * counts[0] + expert(1) * counts[1]; + assert_eq!(plan.cards[1].bytes(), selected + rows); + assert!(plan.violations().is_empty(), "{downs:?}, {room}"); + } + } + } + + /// A malformed descriptor must refuse a plan and a later selected-reserve + /// read, not wrap to a small scratch or masquerade as a fixed-reserve card. + #[test] + fn a_tier_refuses_invalid_scratch_descriptors_before_placement() { + use crate::accepts::{Program, routed_rule}; + use crate::kernels::{DownAct, DownForms}; + use workstation::{TIER_BATCH_RESERVE, TierLegScratch, try_tier_leg_batch_bytes_of}; + + let forms = DownForms::of(DownAct::Q8Act); + let batch = try_tier_leg_batch_bytes_of(256, 256, 1, (1, 1), forms).expect("valid batch"); + let good = batch.leg.expect("a leg descriptor"); + let mut model = legged(&[[GgmlType::Q5_K; 3]]); + model.experts_used = 1; + for t in model + .tensors + .iter_mut() + .filter(|t| t.role == Role::RoutedExperts) + { + t.dims[1] = 256; + t.file_bytes *= 64; + } + let machine = |leg| { + let mut tier = bytes_card("tier", 1 << 30, 0..0); + tier.reserves + .push((TIER_BATCH_RESERVE.into(), batch.card().unwrap())); + tier.tier_leg = Some(leg); + Machine { + cards: vec![bytes_card("stage", HEAD, 0..1)], + tiers: vec![tier], + nodes: vec![Node { + host: host(), + unified: None, + }], + } + }; + let valid = machine(good); + let rule = routed_rule(Program::Qwen35Body); + let plan = plan_routed(&model, &valid, 1, &NoKv, &PlanLevers::default(), rule) + .expect("valid descriptor plans"); + for leg in [ + TierLegScratch { + chunk: 1 << 61, + n_used: 8, + ..good + }, + TierLegScratch { chunk: 0, ..good }, + TierLegScratch { ff: 0, ..good }, + TierLegScratch { n_embd: 0, ..good }, + ] { + let bad = machine(leg); + assert!( + matches!( + plan_routed(&model, &bad, 1, &NoKv, &PlanLevers::default(), rule), + Err(PlacementError::Metadata { key, .. }) if key == "tier leg scratch" + ), + "{leg:?}" + ); + let mut stale = plan.clone(); + stale.machine = &bad; + assert!( + matches!( + stale.card_reserve(1, TIER_BATCH_RESERVE), + Err(PlacementError::Metadata { key, .. }) if key == "tier leg scratch" + ), + "{leg:?}" + ); + assert!( + matches!( + try_tier_leg_batch_bytes_of( + leg.n_embd, leg.ff, leg.n_used, (leg.chunk, 1), leg.downs, + ), + Err(PlacementError::Metadata { key, .. }) if key == "tier leg scratch" + ), + "{leg:?}" + ); + } + assert!(try_tier_leg_batch_bytes_of(256, 256, 8, (1, 1 << 61), forms).is_err()); + let inconsistent = workstation::TierBatchBytes { + scratch: 0, + ..batch + }; + assert!(matches!( + inconsistent.card(), + Err(PlacementError::Metadata { key, .. }) if key == "tier leg batch" + )); + } + // PIN(2026-10-22): the Qwen3.6 leg runs a Q8_0 down and (once the leg has the // arm) a Q6_K gate and up through the common leg, so the layers that held those // types hold types it still has no GEMM for (an MXFP4 down, an IQ2_XS gate and diff --git a/crates/placement/src/placement/card_budget.rs b/crates/placement/src/placement/card_budget.rs index 3f8d2e734..ce33830d7 100644 --- a/crates/placement/src/placement/card_budget.rs +++ b/crates/placement/src/placement/card_budget.rs @@ -225,6 +225,7 @@ mod tests { token_embedding: false, reserves: Vec::new(), reserve_only: false, + tier_leg: None, } } diff --git a/crates/placement/src/placement/nodes_scale.rs b/crates/placement/src/placement/nodes_scale.rs index 94e8830fe..b85b290b9 100644 --- a/crates/placement/src/placement/nodes_scale.rs +++ b/crates/placement/src/placement/nodes_scale.rs @@ -283,6 +283,7 @@ fn census_budgets_keep_each_node_and_stage_terms() { staging: 123, scratch: 456, host: 789, + leg: None, }, ) .expect("three independent nodes"); @@ -339,6 +340,7 @@ fn cases() -> &'static [Case] { staging: 0, scratch: 0, host: 0, + leg: None, }, ) .expect("two nodes with no batch bytes"); diff --git a/crates/placement/src/placement/qwen38_cards.rs b/crates/placement/src/placement/qwen38_cards.rs index 80fd5697f..c5d122985 100644 --- a/crates/placement/src/placement/qwen38_cards.rs +++ b/crates/placement/src/placement/qwen38_cards.rs @@ -255,6 +255,7 @@ const CHECKPOINT_38: u64 = 36 * 4_093_584 + 17 * 4 * 2560 * 4; fn stage(spec: CardSpec) -> Card { Card { name: spec.name.to_string(), + tier_leg: None, device: spec.device, usable_bytes: spec.usable_bytes(), context_bytes: CONTEXT, @@ -303,6 +304,7 @@ fn machine_bp(stage_spec: CardSpec, tier: CardSpec) -> Machine { cards: vec![s], tiers: vec![Card { name: tier.name.to_string(), + tier_leg: None, device: tier.device, usable_bytes: tier.usable_bytes(), context_bytes: CONTEXT, diff --git a/crates/placement/src/placement/workstation.rs b/crates/placement/src/placement/workstation.rs index 6b27baac9..5ab150537 100644 --- a/crates/placement/src/placement/workstation.rs +++ b/crates/placement/src/placement/workstation.rs @@ -19,7 +19,9 @@ use super::nodes::{Fabric, Nodes}; use super::{ Card, Device, Host, HostTotals, Machine, Node, PlacementError, Plan, Reserve, UnifiedPool, }; -use crate::kernels::{DownAct, DownForms, leg_bytes, q8_act_bytes}; +use crate::kernels::{ + DownAct, DownForms, leg_bytes, q8_act_bytes, try_leg_bytes, try_q8_act_bytes, +}; pub const MIB: u64 = 1 << 20; @@ -182,6 +184,7 @@ pub fn model_v41() -> String { fn card(spec: CardSpec, layers: Range, head: bool) -> Card { Card { name: spec.name.to_string(), + tier_leg: None, device: spec.device, usable_bytes: spec.usable_bytes(), context_bytes: CONTEXT, @@ -210,6 +213,7 @@ pub fn tier_card(spec: CardSpec, batch: TierBatchBytes) -> Card { .card() .unwrap_or_else(|error| panic!("tier batch card reserve: {error}")), )); + tier.tier_leg = batch.leg; tier } @@ -373,6 +377,7 @@ pub const NO_TIER_BATCH: TierBatchBytes = TierBatchBytes { staging: 0, scratch: 0, host: 0, + leg: None, }; /// What the expert tier's prompt-batch service is called as a reserve: on @@ -401,11 +406,21 @@ pub struct TierBatchBytes { pub staging: u64, pub scratch: u64, pub host: u64, + /// A common leg's candidate-dependent scratch; `scratch` is its bound. + pub leg: Option, } impl TierBatchBytes { - /// The tier card's part; refuses a sum past u64. + /// The tier card's part; refuses an invalid leg bound or a sum past u64. pub fn card(&self) -> Result { + if let Some(leg) = self.leg + && leg.bytes(leg.downs)? != self.scratch + { + return Err(PlacementError::Metadata { + key: "tier leg batch".into(), + detail: "scratch reserve differs from the descriptor's bound".into(), + }); + } self.staging .checked_add(self.scratch) .ok_or(PlacementError::TierBatchCard { @@ -415,6 +430,42 @@ impl TierBatchBytes { } } +/// The common leg's scratch geometry and the descriptor's admitted forms. +/// Placement prices its candidate's selected forms jointly with expert rows. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TierLegScratch { + pub n_embd: u64, + pub ff: u64, + pub n_used: u64, + pub chunk: u64, + pub downs: DownForms, +} + +impl TierLegScratch { + /// Scratch for exactly these selected forms, refusing invalid geometry + /// and arithmetic overflow before an allocation or placement candidate. + pub fn bytes(self, downs: DownForms) -> Result { + let invalid = || PlacementError::Metadata { + key: "tier leg scratch".into(), + detail: format!("invalid or overflowing geometry: {self:?}, selected {downs:?}"), + }; + if self.chunk == 0 + || self.n_used == 0 + || self.ff == 0 + || self.n_embd == 0 + || [ + crate::kernels::DownAct::Q8Act, + crate::kernels::DownAct::Q8Blocks32, + ] + .into_iter() + .any(|act| downs.holds(act) && !self.downs.holds(act)) + { + return Err(invalid()); + } + try_leg_bytes(self.chunk, self.n_used, self.n_embd, self.ff, downs).ok_or_else(invalid) + } +} + /// Experts the tile path's bucket table takes a layer at most, and the /// columns a tile holds: the tile path's shapes, which size its scratch. const TILE_BUCKET_EXPERTS: u64 = 1024; @@ -425,8 +476,38 @@ const TILE_COLS: u64 = 8; /// activations, the tier places, their q8_1 form, the down outputs by slot. #[must_use] pub const fn tier_batch_staging_bytes(n_embd: u64, n_used: u64, cols: u64) -> u64 { - let slots = cols * n_used; - 4 * cols * n_embd + 4 * slots + q8_act_bytes(cols, n_embd) + 4 * slots * n_embd + match try_tier_batch_staging_bytes(n_embd, n_used, cols) { + Some(bytes) => bytes, + None => panic!("tier_batch_staging_bytes: staging bytes overflow u64"), + } +} + +const fn try_tier_batch_staging_bytes(n_embd: u64, n_used: u64, cols: u64) -> Option { + let Some(slots) = cols.checked_mul(n_used) else { + return None; + }; + let Some(places) = 4u64.checked_mul(slots) else { + return None; + }; + let Some(down) = places.checked_mul(n_embd) else { + return None; + }; + let Some(rows) = 4u64.checked_mul(cols) else { + return None; + }; + let Some(rows) = rows.checked_mul(n_embd) else { + return None; + }; + let Some(bytes) = rows.checked_add(places) else { + return None; + }; + let Some(q8) = try_q8_act_bytes(cols, n_embd) else { + return None; + }; + let Some(bytes) = bytes.checked_add(q8) else { + return None; + }; + bytes.checked_add(down) } /// The tile path's scratch for blocks of up to `slots` slots of rows of @@ -474,9 +555,9 @@ pub const fn walk_a_tile_scratch_bytes(n_embd: u64, ff: u64, n_used: u64, cols: /// The tier's chunk scratch for blocks of rows of `n_embd`, `n_used` slots a /// token, through experts of `ff`, cut in chunks of `chunk` tokens: one run of -/// the card leg's buffers (`bloomery_gpu::card_leg::LegBufs`, the GLM-5.3-Flash -/// tier) with its slots' columns in the 128-value q8_1 form the tier's downs -/// read ([`leg_bytes`]), in place of the tile path's scratch. +/// the card leg's buffers (`bloomery_gpu::card_leg::LegBufs`) with its slots' +/// columns in the 128-value form ([`leg_bytes`]), in place of the tile path's +/// scratch. Selected-form callers use [`try_tier_leg_batch_bytes_of`]. #[must_use] pub const fn tier_chunk_scratch_bytes(n_embd: u64, ff: u64, n_used: u64, chunk: u64) -> u64 { leg_bytes(chunk, n_used, n_embd, ff, DownForms::of(DownAct::Q8Act)) @@ -486,8 +567,26 @@ pub const fn tier_chunk_scratch_bytes(n_embd: u64, ff: u64, n_used: u64, chunk: /// rows the tier hands back and the places it reads. #[must_use] pub const fn tier_batch_host_bytes(n_embd: u64, n_used: u64, cols: u64) -> u64 { - let slots = cols * n_used; - 2 * (4 * slots * n_embd + 4 * slots) + match try_tier_batch_host_bytes(n_embd, n_used, cols) { + Some(bytes) => bytes, + None => panic!("tier_batch_host_bytes: host bytes overflow u64"), + } +} + +const fn try_tier_batch_host_bytes(n_embd: u64, n_used: u64, cols: u64) -> Option { + let Some(slots) = cols.checked_mul(n_used) else { + return None; + }; + let Some(places) = 4u64.checked_mul(slots) else { + return None; + }; + let Some(down) = places.checked_mul(n_embd) else { + return None; + }; + let Some(rows_and_places) = down.checked_add(places) else { + return None; + }; + 2u64.checked_mul(rows_and_places) } /// The expert tier's prompt-batch bytes for blocks of up to `cols` tokens of @@ -498,22 +597,21 @@ pub const fn tier_batch_bytes(n_embd: u64, ff: u64, n_used: u64, cols: u64) -> T staging: tier_batch_staging_bytes(n_embd, n_used, cols), scratch: tier_block_scratch_bytes(n_embd, ff, cols * n_used), host: tier_batch_host_bytes(n_embd, n_used, cols), + leg: None, } } -/// [`tier_batch_bytes`] for a tier on the common card leg (`bloomery_gpu::host::leg_tier`): the -/// scratch is one run of the leg's buffers for `chunk` tokens ([`tier_chunk_scratch_bytes`]), in -/// place of the tile path's. The one formula of those bytes for every family whose tier is the card -/// leg; a family names its own `chunk` and widths. -#[must_use] -pub const fn tier_leg_batch_bytes( +/// [`tier_batch_bytes`] for a candidate-selected common leg in the 128-value form. +/// +/// Refuses invalid geometry or overflowing byte counts before placement. +pub fn tier_leg_batch_bytes( n_embd: u64, ff: u64, n_used: u64, chunk: u64, cols: u64, -) -> TierBatchBytes { - tier_leg_batch_bytes_of( +) -> Result { + try_tier_leg_batch_bytes_of( n_embd, ff, n_used, @@ -522,22 +620,42 @@ pub const fn tier_leg_batch_bytes( ) } -/// [`tier_leg_batch_bytes`] for a tier whose layers' downs read the forms `downs`: the leg's -/// buffers hold the slots' columns in each of them (`bloomery_gpu::card_leg::LegBufs::for_downs`, -/// [`leg_bytes`]), as an NVFP4 down's 32-value blocks. -#[must_use] -pub const fn tier_leg_batch_bytes_of( +/// The common leg's staging, host rows and selected-form scratch, refusing +/// invalid geometry or overflowing byte counts before placement. +pub fn try_tier_leg_batch_bytes_of( n_embd: u64, ff: u64, n_used: u64, (chunk, cols): (u64, u64), downs: DownForms, -) -> TierBatchBytes { - TierBatchBytes { - staging: tier_batch_staging_bytes(n_embd, n_used, cols), - scratch: leg_bytes(chunk, n_used, n_embd, ff, downs), - host: tier_batch_host_bytes(n_embd, n_used, cols), - } +) -> Result { + let leg = TierLegScratch { + n_embd, + ff, + n_used, + chunk, + downs, + }; + let scratch = leg.bytes(downs)?; + let counted = || -> Option<(u64, u64)> { + if cols == 0 { + return None; + } + Some(( + try_tier_batch_staging_bytes(n_embd, n_used, cols)?, + try_tier_batch_host_bytes(n_embd, n_used, cols)?, + )) + }; + let (staging, host) = counted().ok_or_else(|| PlacementError::Metadata { + key: "tier leg batch".into(), + detail: format!("invalid or overflowing staging: {leg:?}, columns {cols}"), + })?; + Ok(TierBatchBytes { + staging, + scratch, + host, + leg: Some(leg), + }) } /// Plan (b′): the A6000 runs all `layers` and the head as in [`plan_a`]; @@ -629,6 +747,7 @@ pub fn plan_node_tiers( remote.cards[0] .reserves .extend(tier_card(spec, batch).reserves); + remote.cards[0].tier_leg = batch.leg; machines.push(remote); } Ok(machines) @@ -811,6 +930,7 @@ pub(crate) fn spark_nodes(n: usize, layers: usize, fabric: Fabric) -> Nodes { staging: 0, scratch: 0, host: 0, + leg: None, }; let mut machines = plan_node_tiers(layers, spec, &vec![spec; n - 1], batch) .expect("synthetic nodes with no batch bytes"); @@ -1701,6 +1821,7 @@ mod tests { staging: 1, scratch: 2, host: 3, + leg: None, }, ]; for layers in [0, 1, 20, 43, 47, 48, 61, 78] { @@ -1856,6 +1977,7 @@ mod tests { staging: u64::MAX, scratch: 1, host: 0, + leg: None, }; for result in [ plan_tiers(4, A6000, &[RTX_3090], None, huge).map(|_| ()), @@ -1906,6 +2028,7 @@ mod tests { staging: 10, scratch: 5, host: 100, + leg: None, }; let machines = plan_node_tiers(4, specs[0], &specs[1..], batch) .expect("four nodes of the same card name and ordinal"); @@ -1950,6 +2073,7 @@ mod tests { staging: 0, scratch: 0, host: u64::MAX / 2, + leg: None, }; let remotes = [gb10(80 << 30), gb10(60 << 30)]; let machines = @@ -1985,6 +2109,7 @@ mod tests { staging: 10, scratch: 5, host: 100, + leg: None, }; let m = plan_tiers(4, A6000, &[t1, t2], None, batch).expect("two tiers"); let rows: Vec<&(String, u64)> = m @@ -2065,6 +2190,7 @@ mod tests { assert_eq!(counted_arena_bytes(&m.cards[0]), 7, "{}", spec.name); let want = Card { name: spec.name.to_string(), + tier_leg: None, device: spec.device, usable_bytes: spec.usable_bytes(), context_bytes: CONTEXT, @@ -2112,6 +2238,7 @@ mod tests { staging: u64::MAX, scratch: 1, host: 0, + leg: None, }, ); } @@ -2131,6 +2258,7 @@ mod tests { staging: u64::MAX - 1, scratch: 1, host: 0, + leg: None, }; assert_eq!(edge.card().expect("the sum fits u64"), u64::MAX); assert!(matches!( diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 5a4efc3c7..837e44015 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -2,8 +2,8 @@ Design snapshot: **`e53cabebbb9b16249c81b7c26a026da1670f7d9d`**, requested by [#632](https://github.com/midagedev/bloomery/issues/632). The source-gap observations below describe -that snapshot. This component tree has a header-only reader, shared tier-handoff work and the ordinary -IQ4_XS shared site, but no Flint body, app session or serve seat. Primitive GPU evidence is not Flint model admission. +that snapshot. This component tree has a header-only reader, shared tier-handoff work, the ordinary +IQ4_XS shared site and selected common-leg work, but no Flint body, app session or serve seat. Primitive GPU evidence is not Flint model admission. ## Six-slot handoff closure checks — 2026-10-26 @@ -89,31 +89,6 @@ Each device slice needs its numerical/fault gates and the maintainer's offered h Legacy metadata booleans share `RouterNorm::from_legacy_bool`: false keeps scores and true uses the existing epsilon-added selected sum. Flint's distinct floor mapping stays in its reader. -### Six-slot F32 tier handoff implementation - -`HandoffKernels::enqueue_handoff_tier` now selects a genuine six-slot F32 entry -beside the existing four- and eight-slot entries. Both host and tier layouts must have the -same supported slot count and activation width; compressed tier images remain -refused. The entry uses the existing copy/place/fault helpers and the existing -subsequent go/publication protocol; no Flint router or Program is registered. - -`hw_f32_tier_handoffs_conserve_consumed_routes`, owned by -`gate-gpu-ds41-lib`, consumes the four-, six- and eight-slot handoffs' weights and emitted -places through the production tier accumulator and adds the host sum. It covers partial -widths, invalid ids, poisoned non-owning rows, trailing sentinels and changed replay -weights. Its host sum is precomputed and activation images are checked by -readback: it is not expert execution or two-card completion/publication proof. -Earlier integrated-source results do not verify this branch after its merge with current -main; the final-source hardware result must be recorded separately. - -The new `ffn_handoff_6_tier` entry needs add-class PTX/JIT/resource and -owning-gate evidence; the four-slot, eight-slot and other existing entries must remain -unchanged. The cc12.1 tables pin it at the cc8.6 scan's values until an observed NVIDIA JIT run. -Actual six-slot body/tier and two-card completion remain integration work, not -capabilities inferred from this primitive. -`check-ptx-names` is green with those pinned rows; a GB10 scan replaces them with the -observed spill/JIT-local measurements. Offline no-JIT scans cannot supply that column. - ### Ordinary IQ4_XS shared-site implementation IQ4_XS now has an ordinary file-layout/dense entry in QROWS and a common site family. @@ -164,6 +139,124 @@ later diagnostic, test or documentation changes. Wider affected real-model, placement and residency coverage remains explicit; this does not establish a Flint oracle, physical cc12.1, ROCm or performance. +### Selected expert arms and tier scratch + +Common card and tier wrappers resolve `LegArms` from their selected `Program`, rather than assuming +every family has one gate/up type or one down activation form. Their resident layers determine the +union of `Q8Act` and `Q8Blocks32` buffers. Plane layouts remain restricted to the selected program's +arms; Solar's host-only triples do not enlarge its card scratch. + +Placement charges a tier's selected down forms jointly with its expert count. The same kernels-owned, +checked sizing arithmetic is used by allocation and planning; malformed or overflowing public scratch +descriptors are refused by name. The plan records the selected reserve, and loaders check that amount +instead of the machine descriptor's upper bound. No always-both allocation or reserve-equality waiver +is used. + +The wrapper fixture loads each expert part through the selected program's `routed_rule` format, +`Segment::buffer_bytes` and `Weights::load_rows`, including selected-only IQ3_S. Its mixed loads +compare real step, pass, batch and tier allocations and rows against the common leg. A mismatched +gate/up pair remains refused; Solar's admitted Q6_K down is not a rejection case. Tier activation +selection remains the family's contract, independently of the program-owned stack admission. + +The 2026-10-26 cutover removes the obsolete always-Q8 tier-reserve regression: +selected mixed-layer and mismatched-gate/up coverage remains in +`tier_scratch_follows_selected_mixed_layers`, with allocation and budget boundaries +covered by the placement and GPU wrapper consumers. The MiniMax planner test retains +its reserve and idle-tier behavior checks rather than comparing descriptor representation +against a different constructor. +Host-only projections intentionally have no card totals: fixed reserves remain owned by their machine. +Only a dynamic tier-leg descriptor requires the plan's selected card totals. The existing node paging +budget regression exercises this distinction without a GPU, including OS reserve and constrained host room. + +`tier_leg_batch_bytes` and the explicit-form `try_tier_leg_batch_bytes_of` return +`Result` and construct candidate-selected descriptors. +An empty tier selects no down form, matching `LegTier`'s actual allocation instead of +advertising unused Q8Act scratch. Qwen3/Qwen35, MiMo and Muse use this checked path. +Qwen3/Qwen35 and MiniMax derive their descriptor bounds from the described model's +admitted forms through `accepts::leg_down_forms`, as GLM5Next and Solar do. +The Qwen public wrappers take `PlanInputs`, not geometry alone: a supported +Q8_0, Q5_1 or IQ4_NL candidate must not fail the whole plan just because its +32-value form remains unselected at the tier's budget boundary. +No fallback format can admit a mismatched triple. +The checked constructors and non-leg paths share staging and host byte formulas. +Malformed geometry and overflow are refused rather than panicking; a dense Qwen3.6 +request for an expert tier is a named metadata refusal, not zero expert geometry. +Separate tile/route-based fixed-reserve paths remain unchanged. + +The planner's selected scratch calculation has one union/byte rule. Fill prepares +per-layer forms once for dynamic tier-leg cards, including multiple forms in one +layer, and reuses them with its cached footprint instead of rescanning tensors for +each candidate. Fixed-reserve cards need no selected-form cache. + +The Solar budget regression starts from a typed mixed GGUF description and exercises +the planner's one-byte refusal/success boundary without overwriting its described +prompt reserve. With byte granules (no reusable dense padding), the exact dense-and-prompt +budget leaves no room for expert weights. +The real mixed-wrapper hardware case also checks allocation of a configured empty tier. + +Historical exact-source proof at `3071a9f0d8ed4773692e0c040dece222b90f6bec`: +GPU library418 passed plus10 held passed, verbs1 passed, and all1657 canonical entries +identical to main15778e2c (4bins/8carriers, non-JIT rows equal). +The repair at `c26025b1fb9fad1af399401d27cf709005736ba0` then executed the extended +real CUDA mixed/empty-tier consumer:1 passed,0 failed,0 ignored,427 filtered,24.18s. +Its fresh4bin/8carrier canonical comparison again found all1657entries identical to main15778e2c, +with non-JIT rows equal. Native formatting/Clippy passed (warning3=ratchet3); model557 and +placement202 passed,0 failed,5 model tests ignored,0 filtered. + +The historical component contract source `7e37ef9a6ebbd766032ee521a0ee16e25f163851` +executed the real CUDA mixed/empty-tier consumer on RTX3090 cc8.6:1 passed, +0 failed/ignored,427 filtered,23.98s. Its metadata-only +`a_dense_header_plans_through_qwen35_on` frontend consumer also passed1 case, +including the named dense-header expert-tier refusal; that is not a second GPU case. +The review code passed native formatting/all-target Clippy (warning3=ratchet3), +chain23, GGUF67, model557 and placement202 (five model cases ignored,0 filtered). +An exact-source7e37ef9a canonical comparison against main15778e2c found all1657 +entries identical across4bins/8carriers and non-JIT rows equal; it did not compare +driver-JIT resource rows. The older full-library counts above remain historical, +not a new full-library run of the review repairs. + +The component subsequently merges pinned upstream `cb8d92ebddd1ce1a325a15450c06a01f57ecf88c`. +That reconciliation retains upstream's GLM mixer-form consumers alongside this component's +checked tier-byte API; the explicit-form API has no input-scale flag. The GPU results +above belong to their named historical sources, not the merged source. + +At merged source `c38d603ca1d33c6c2bede63aba102199715fb9ef`, native formatting and +all-target Clippy passed, as did chain23, GGUF67, model559 and placement202 +(five model cases ignored, zero failed/filtered). The actual oxide-linked +`a_dense_header_plans_through_qwen35_on` frontend consumer passed1 case, +0 failed/ignored,6 filtered, including the named expert-tier refusal. +This is metadata-only host execution, not fresh NVIDIA execution of the merged source +or a whole-model numerical result. + +The host-only public planner consumer covers one-byte boundaries before the first resident layer +and before adding another down form, plus descriptor overflow refusal. These and the CUDA consumer +are not a whole-model oracle or every affected model/residency gate; canonical equality does not +waive changed host/allocation consumers. Flint's `Program` admission and body/app/seat remain absent. +### Six-slot F32 tier handoff implementation + +`HandoffKernels::enqueue_handoff_tier` now selects a genuine six-slot F32 entry +beside the existing four- and eight-slot entries. Both host and tier layouts must have the +same supported slot count and activation width; compressed tier images remain +refused. The entry uses the existing copy/place/fault helpers and the existing +subsequent go/publication protocol; no Flint router or Program is registered. + +`hw_f32_tier_handoffs_conserve_consumed_routes`, owned by +`gate-gpu-ds41-lib`, consumes the four-, six- and eight-slot handoffs' weights and emitted +places through the production tier accumulator and adds the host sum. It covers partial +widths, invalid ids, poisoned non-owning rows, trailing sentinels and changed replay +weights. Its host sum is precomputed and activation images are checked by +readback: it is not expert execution or two-card completion/publication proof. +Earlier integrated-source results do not verify this branch after its merge with current +main; the final-source hardware result must be recorded separately. + +The new `ffn_handoff_6_tier` entry needs add-class PTX/JIT/resource and +owning-gate evidence; the four-slot, eight-slot and other existing entries must remain +unchanged. The cc12.1 tables pin it at the cc8.6 scan's values until an observed NVIDIA JIT run. +Actual six-slot body/tier and two-card completion remain integration work, not +capabilities inferred from this primitive. +`check-ptx-names` is green with those pinned rows; a GB10 scan replaces them with the +observed spill/JIT-local measurements. Offline no-JIT scans cannot supply that column. + ## 1. Sources and observed artifact