From a4f3919e445f7d1b3fd3c739731a75dda2adea71 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:34:14 +0900 Subject: [PATCH 01/19] gpu: size mixed expert scratch from selected resident forms --- crates/gpu-deepseek41/src/body.rs | 2 +- .../gpu-gates/src/bin/gate_glm5next_nodes.rs | 2 +- .../src/bin/gate_glm5next_twocard.rs | 2 +- crates/gpu-gates/src/bin/gate_load_v41.rs | 1 + .../gpu-gates/src/bin/gate_solar2_twocard.rs | 2 +- crates/gpu-gates/src/bin/generate_glm5next.rs | 2 +- .../src/bin/shared/glm5next_place.rs | 11 +- .../src/bin/shared/routed_legs/glm5next.rs | 3 +- .../src/bin/shared/serve_seats/glm.rs | 2 +- .../src/bin/shared/serve_seats/hybrid.rs | 15 +- .../src/bin/shared/serve_seats/solar2.rs | 9 +- .../src/bin/shared/serve_seats/tiered.rs | 12 +- .../gpu-gates/src/bin/shared/solar2_seat.rs | 2 +- crates/gpu-gates/src/generate.rs | 1 + crates/gpu-glm5next/src/tier.rs | 1 + crates/gpu-mimo2/src/card.rs | 1 + crates/gpu-minimaxm3/src/ffn.rs | 1 + crates/gpu-solar2/src/ffn.rs | 1 + crates/gpu-solar2/src/tier.rs | 1 + crates/gpu/src/arch/qwen3moe/body38.rs | 2 +- crates/gpu/src/card_experts.rs | 374 +++++++++++++++++- crates/gpu/src/card_leg.rs | 18 +- crates/gpu/src/host/batch.rs | 28 +- crates/gpu/src/host/leg_tier.rs | 43 +- crates/gpu/src/host/node.rs | 14 +- crates/model/src/arch/deepseek41/kv.rs | 1 + crates/model/src/arch/glm5next/place.rs | 134 +++---- crates/model/src/arch/qwen35moe/place.rs | 6 + crates/model/src/arch/solar2/place.rs | 157 ++++++-- crates/model/tests/placement.rs | 1 + crates/model/tests/qwen4exp_mtp_reserve.rs | 1 + crates/model/tests/solar_open2_meta.rs | 1 + crates/placement/src/accepts.rs | 37 ++ crates/placement/src/kernels.rs | 157 +++++++- crates/placement/src/placement.rs | 349 ++++++++++++++-- crates/placement/src/placement/nodes_scale.rs | 2 + .../placement/src/placement/qwen38_cards.rs | 2 + crates/placement/src/placement/workstation.rs | 125 +++++- docs/research/flint-attach.md | 19 + 39 files changed, 1321 insertions(+), 221 deletions(-) diff --git a/crates/gpu-deepseek41/src/body.rs b/crates/gpu-deepseek41/src/body.rs index 0ef84a357..051d00605 100644 --- a/crates/gpu-deepseek41/src/body.rs +++ b/crates/gpu-deepseek41/src/body.rs @@ -2563,7 +2563,7 @@ impl Body { // The tiers' batch staging is reserved on their cards by the plan, // so it is made at load, not at the first prompt. hybrid.prepare_batch(gpu.context(), prefill::T_MAX)?; - hybrid.check_tier_reserves(plan.machine, &cards)?; + hybrid.check_tier_reserves(plan, &cards)?; Some(TierPiece::new(gpu, hybrid.slots(), PAIR_ROWS)?) }; let ced = ced::Ced::new(&CedLayer::table(hp), seq.ring_rows(), cfg.ced); diff --git a/crates/gpu-gates/src/bin/gate_glm5next_nodes.rs b/crates/gpu-gates/src/bin/gate_glm5next_nodes.rs index 8b9f34dc6..0845bd66a 100644 --- a/crates/gpu-gates/src/bin/gate_glm5next_nodes.rs +++ b/crates/gpu-gates/src/bin/gate_glm5next_nodes.rs @@ -30,7 +30,7 @@ mod gate { let file = Split::open(&path)?; let inputs = PlanInputs::describe(&file)?; let (mut machine, node) = - crate::node_history::machine(inputs.model.layers, tier_batch(&inputs.hp))?; + crate::node_history::machine(inputs.model.layers, tier_batch(&inputs)?)?; reserve_checkpoints( &mut machine, Seqs::priced(1, checkpoint_price(&inputs)?), diff --git a/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs b/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs index 5451922c0..1710e416d 100644 --- a/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs +++ b/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs @@ -515,7 +515,7 @@ mod gate { fn bp_machine(inputs: &PlanInputs) -> Result Machine + Copy, GateError> { glm_machine( Place::Bp, - Some(place::tier_batch(&inputs.hp)), + Some(place::tier_batch(inputs)?), 1, checkpoint_price(inputs)?, u64::try_from(CTX)?, diff --git a/crates/gpu-gates/src/bin/gate_load_v41.rs b/crates/gpu-gates/src/bin/gate_load_v41.rs index ddd187806..e9304096d 100644 --- a/crates/gpu-gates/src/bin/gate_load_v41.rs +++ b/crates/gpu-gates/src/bin/gate_load_v41.rs @@ -242,6 +242,7 @@ mod gate { staging: 0, scratch: 0, host: 0, + leg: None, }, ) .map_err(|e| format!("plan (b): {e}"))?; diff --git a/crates/gpu-gates/src/bin/gate_solar2_twocard.rs b/crates/gpu-gates/src/bin/gate_solar2_twocard.rs index dae44759b..9d413942a 100644 --- a/crates/gpu-gates/src/bin/gate_solar2_twocard.rs +++ b/crates/gpu-gates/src/bin/gate_solar2_twocard.rs @@ -313,7 +313,7 @@ mod gate { let layers = inputs.model.layers; let bp = Place::Bp .on_host()? - .machine(None, Some(place::tier_batch(&inputs.hp)))?; + .machine(None, Some(place::tier_batch(&inputs)?))?; let two = bp(layers); let plan = inputs.plan(&two, u64::try_from(CTX)?, &cfg.place)?; let budget = cfg diff --git a/crates/gpu-gates/src/bin/generate_glm5next.rs b/crates/gpu-gates/src/bin/generate_glm5next.rs index 27a6976b5..26a907960 100644 --- a/crates/gpu-gates/src/bin/generate_glm5next.rs +++ b/crates/gpu-gates/src/bin/generate_glm5next.rs @@ -761,7 +761,7 @@ mod cli { Some(lane_at(placement)?) }; let place = placement.name(); - let tier_batch = glm_place::tier_batch(placement, &inputs.hp); + let tier_batch = glm_place::tier_batch(placement, &inputs)?; let machine = glm_machine( placement, tier_batch, diff --git a/crates/gpu-gates/src/bin/shared/glm5next_place.rs b/crates/gpu-gates/src/bin/shared/glm5next_place.rs index ac6a3dd03..36f0e5eed 100644 --- a/crates/gpu-gates/src/bin/shared/glm5next_place.rs +++ b/crates/gpu-gates/src/bin/shared/glm5next_place.rs @@ -10,7 +10,6 @@ use bloomery_gpu_gates::beside::{self, Beside}; use bloomery_gpu_gates::generate::{Place, TierRule, choose_ctx_of}; use bloomery_gpu_gates::residency38::glm_machine; use bloomery_gpu_glm5next::checkpoint_price; -use model::arch::glm5next::hparams::Hparams; use model::arch::glm5next::place::{ KdaLanes, NextnInputs, ORACLE_POSITIONS, PlaceError, PlanInputs, }; @@ -41,10 +40,12 @@ pub fn parse(v: &str) -> Result { } /// The expert tier's prompt-batch bytes `place`'s plan reserves for the file -/// of hyperparameters `hp` (`model::arch::glm5next::place::tier_batch`), +/// described by `inputs` (`model::arch::glm5next::place::tier_batch`), /// which `Place::machine` takes; `None` under a placement with no tier card. -pub fn tier_batch(place: Place, hp: &Hparams) -> Option { - (!place.tier_cards().is_empty()).then(|| model::arch::glm5next::place::tier_batch(hp)) +pub fn tier_batch(place: Place, inputs: &PlanInputs) -> Result, GateError> { + Ok((!place.tier_cards().is_empty()) + .then(|| model::arch::glm5next::place::tier_batch(inputs)) + .transpose()?) } /// The experts `place`'s tier cards hold in its plan at `ctx` positions @@ -65,7 +66,7 @@ pub fn tier_experts( ) -> Result { let machine = glm_machine( place, - tier_batch(place, &inputs.hp), + tier_batch(place, inputs)?, slots, checkpoint_price(inputs)?, ctx, diff --git a/crates/gpu-gates/src/bin/shared/routed_legs/glm5next.rs b/crates/gpu-gates/src/bin/shared/routed_legs/glm5next.rs index 42307899b..5db7eebca 100644 --- a/crates/gpu-gates/src/bin/shared/routed_legs/glm5next.rs +++ b/crates/gpu-gates/src/bin/shared/routed_legs/glm5next.rs @@ -172,7 +172,7 @@ fn machine_of( slots: usize, ) -> Result { let price = checkpoint_price(inputs)?; - let tier_batch = glmplace::tier_batch(place, &inputs.hp); + let tier_batch = glmplace::tier_batch(place, inputs)?; Ok(glm_machine(place, tier_batch, slots, price, ctx)?( inputs.model.layers, )) @@ -242,7 +242,6 @@ fn place_of(p: &PlaceError) -> Outcome { PlaceError::Placement(e) => return refused_of(e), PlaceError::PastOracle { .. } => "past-oracle", PlaceError::FrontOver { .. } => "front-over", - PlaceError::TierLeg { .. } => "tier-leg", PlaceError::Broken(_) | PlaceError::NextnCards { .. } | PlaceError::Slots { .. } => { return Outcome::Red(p.to_string()); } diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs b/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs index 68b6e54f5..9e4e6f3af 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs @@ -503,7 +503,7 @@ impl Placed { (set, slots): (Option, usize), (images, side): (&Images<'_>, Option<&Side>), ) -> Result { - let tier_batch = glm_place::tier_batch(place, &inputs.hp); + let tier_batch = glm_place::tier_batch(place, inputs)?; // Every machine reserves its checkpoints at the slot context it is // laid for: the choice's probes each at their own, the load's at the // context it is granted; and the encoder's reserve, as the printed plan does. diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs b/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs index 0317e82a3..8d16d9516 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs @@ -79,12 +79,16 @@ pub trait Family: Sized + 'static { /// The expert tier's prompt-batch bytes the plan reserves on `place`'s /// tier card ([`Tiered::tier_batch`]); a body that serves no tier card /// reserves none (the default), and its plan refuses the card by name. - fn tier_batch(place: Place, _inputs: &PlanInputs) -> Option { - (!place.tier_cards().is_empty()).then_some(TierBatchBytes { + fn tier_batch( + place: Place, + _inputs: &PlanInputs, + ) -> Result, GateError> { + Ok((!place.tier_cards().is_empty()).then_some(TierBatchBytes { staging: 0, scratch: 0, host: 0, - }) + leg: None, + })) } /// The tags the model's description names for its think span, which the @@ -324,7 +328,10 @@ impl Tiered for Hybrid { F::think_tags(inputs).map(Some) } - fn tier_batch(place: Place, inputs: &PlanInputs) -> Option { + fn tier_batch( + place: Place, + inputs: &PlanInputs, + ) -> Result, GateError> { F::tier_batch(place, inputs) } diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/solar2.rs b/crates/gpu-gates/src/bin/shared/serve_seats/solar2.rs index dc1bd2b21..c53248f87 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/solar2.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/solar2.rs @@ -166,8 +166,13 @@ impl Family for Solar { /// The tier's prompt-batch bytes of the file's widths ([`place::tier_batch`]), reserved on /// the placement's tier card and the host. - fn tier_batch(place: Place, inputs: &PlanInputs) -> Option { - (!place.tier_cards().is_empty()).then(|| place::tier_batch(&inputs.hp)) + fn tier_batch( + place: Place, + inputs: &PlanInputs, + ) -> Result, bloomery_gpu_gates::GateError> { + Ok((!place.tier_cards().is_empty()) + .then(|| place::tier_batch(inputs)) + .transpose()?) } } diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs b/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs index 7681f03e9..e80f07737 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs @@ -101,12 +101,16 @@ pub trait Tiered: Seat + Sized + 'static { /// The expert tier's prompt-batch bytes the plan reserves on `place`'s tier card, for the file /// `inputs` describes; `None` under a placement with no tier card. A body that serves no tier /// card reserves none (the default), and its plan refuses the card by name. - fn tier_batch(place: Place, _inputs: &PlanInputs) -> Option { - (!place.tier_cards().is_empty()).then_some(TierBatchBytes { + fn tier_batch( + place: Place, + _inputs: &PlanInputs, + ) -> Result, GateError> { + Ok((!place.tier_cards().is_empty()).then_some(TierBatchBytes { staging: 0, scratch: 0, host: 0, - }) + leg: None, + })) } /// The think tags the model's description names, if the family splits its reasoning on them. @@ -214,7 +218,7 @@ pub fn run(args: &[String], side: &side_seat::Flags) -> Result Result, GateErro // prompt-batch bytes the file's widths give. let machine = match arm { Arm::A => crate::gate_card::plan_gate(inputs.model.layers), - Arm::Bp => workstation::plan_bp(inputs.model.layers, None, place::tier_batch(&inputs.hp)), + Arm::Bp => workstation::plan_bp(inputs.model.layers, None, place::tier_batch(&inputs)?), }; let place = PlanLevers::from_levers(&levers)?; let plan = inputs.plan(&machine, u64::try_from(ctx)?, &place)?; diff --git a/crates/gpu-gates/src/generate.rs b/crates/gpu-gates/src/generate.rs index 05c90a4f2..49ee7a38c 100644 --- a/crates/gpu-gates/src/generate.rs +++ b/crates/gpu-gates/src/generate.rs @@ -716,6 +716,7 @@ impl Place { staging: 0, scratch: 0, host: 0, + leg: None, }; self.machine(None, (!self.tier_cards().is_empty()).then_some(none)) } diff --git a/crates/gpu-glm5next/src/tier.rs b/crates/gpu-glm5next/src/tier.rs index e540f4ed2..aee07ed13 100644 --- a/crates/gpu-glm5next/src/tier.rs +++ b/crates/gpu-glm5next/src/tier.rs @@ -62,6 +62,7 @@ impl LegFamily for Glm { n_used: hp.n_used, n_embd: hp.n_embd, ff: hp.expert_ff, + program: model::placement::accepts::Program::Glm5nextBody, } } diff --git a/crates/gpu-mimo2/src/card.rs b/crates/gpu-mimo2/src/card.rs index 728908ae7..147124f1a 100644 --- a/crates/gpu-mimo2/src/card.rs +++ b/crates/gpu-mimo2/src/card.rs @@ -26,6 +26,7 @@ fn stacks(l: usize) -> [String; 3] { /// MiMo's card experts: 256 routed experts, 8 a token. const FAMILY: CardFamily = CardFamily { what: "mimo2 CardExperts", + program: model::placement::accepts::Program::Mimo2Body, n_expert: N_EXPERT, n_used: N_USED, stacks, diff --git a/crates/gpu-minimaxm3/src/ffn.rs b/crates/gpu-minimaxm3/src/ffn.rs index 828b0897d..3850f6ba4 100644 --- a/crates/gpu-minimaxm3/src/ffn.rs +++ b/crates/gpu-minimaxm3/src/ffn.rs @@ -20,6 +20,7 @@ pub use bloomery_gpu_deepseek41::shared_routed::{ /// The card experts' family: the file's stack names and the router's counts. pub const CARD: CardFamily = CardFamily { what: "minimaxm3 CardExperts", + program: model::placement::accepts::Program::MiniMaxM3Body, n_expert: N_EXPERT, n_used: N_USED, stacks: names::routed_stacks, diff --git a/crates/gpu-solar2/src/ffn.rs b/crates/gpu-solar2/src/ffn.rs index a0f1589d5..7be370e0a 100644 --- a/crates/gpu-solar2/src/ffn.rs +++ b/crates/gpu-solar2/src/ffn.rs @@ -17,6 +17,7 @@ pub use bloomery_gpu_deepseek41::shared_routed::{GATED_FROM, RoutedDims, back, f /// The card experts' family: the file's stack names and the router's counts. pub const CARD: CardFamily = CardFamily { what: "solar2 CardExperts", + program: model::placement::accepts::Program::Solar2Body, n_expert: N_EXPERT, n_used: N_USED, stacks: names::routed_stacks, diff --git a/crates/gpu-solar2/src/tier.rs b/crates/gpu-solar2/src/tier.rs index 071ce8d20..e23773d93 100644 --- a/crates/gpu-solar2/src/tier.rs +++ b/crates/gpu-solar2/src/tier.rs @@ -64,6 +64,7 @@ impl LegFamily for Solar2 { n_used: experts.n_used, n_embd: inputs.hp.n_embd, ff: experts.ff, + program: model::placement::accepts::Program::Solar2Body, } } diff --git a/crates/gpu/src/arch/qwen3moe/body38.rs b/crates/gpu/src/arch/qwen3moe/body38.rs index 4c1465c29..69e666d79 100644 --- a/crates/gpu/src/arch/qwen3moe/body38.rs +++ b/crates/gpu/src/arch/qwen3moe/body38.rs @@ -1982,7 +1982,7 @@ impl Body38 { hybrid.prepare_batch(gpu.context(), host_cols)?; if tiered { let cards: Vec = tiers.iter().map(|t| t.card).collect(); - hybrid.check_tier_reserves(plan.machine, &cards)?; + hybrid.check_tier_reserves(plan, &cards)?; } // The PLE table's tier is the plan's: the host set built above holds // it on the host; on the NVMe tier its reader reads it. diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index 64282ef47..4542fe4ac 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -31,14 +31,22 @@ use cuda_core::{CudaStream, DeviceBuffer}; use model::arch::models::{Ffn, LayerSpec}; +use model::placement::accepts::{self, LegArms, Program}; +use model::placement::kernels::{self, DownForms}; -use crate::card_leg::{CardLeg, LegBufs, LegRun, LegShape, LegStacks, LegSum, leg_bytes}; +use crate::card_leg::{CardLeg, LegBufs, LegRun, LegShape, LegStacks, LegSum}; use crate::card_slots::{CardSlotKernels, Places}; use crate::hybrid::SlotMap; use crate::kquant::Act; use crate::weights::Weights; use crate::{DeviceTensor, Gpu, GpuError}; +/// The selected program, rather than a union of unrelated launches, admits stacks. +const STACK_ROLES: (&str, &str) = ( + "a resident routed gate or up stack the selected card gate·up arm runs", + "a resident routed down stack the selected card down arm runs", +); + /// What a family fixes of its card experts. #[derive(Clone, Copy)] pub struct CardFamily { @@ -50,6 +58,8 @@ pub struct CardFamily { pub n_used: usize, /// Layer `l`'s gate, up and down stacks' names. pub stacks: fn(usize) -> [String; 3], + /// The program whose selected gate·up and down launches run the stacks. + pub program: Program, } /// The kernels a run launches: the card leg's (its sum stage included), and @@ -57,6 +67,7 @@ pub struct CardFamily { struct CardKernels { leg: CardLeg, places: CardSlotKernels, + arms: LegArms, } impl CardFamily { @@ -89,9 +100,10 @@ impl CardBufs { n: usize, ff: usize, cap: usize, + downs: DownForms, ) -> Result { Ok(CardBufs { - leg: LegBufs::new(stream, family.shape(n, ff, cap))?, + leg: LegBufs::for_downs(stream, family.shape(n, ff, cap), downs)?, acc: DeviceBuffer::zeroed(stream, cap * n)?, pre: DeviceBuffer::zeroed(stream, cap * n)?, }) @@ -217,6 +229,8 @@ pub struct CardExperts { ff: usize, /// The model's width. n: usize, + /// The union of the resident layers' selected down activation forms. + downs: DownForms, /// The one-token step's buffers. step: CardBufs, /// The buffers of a pass of several rows, on a load that runs one. @@ -270,8 +284,13 @@ impl CardExperts { tiers: bool, ) -> Result { let what = family.what; + let arms = accepts::leg_arms(family.program).ok_or(GpuError::state( + what, + "the selected program's card leg arms", + ))?; let mut per = Vec::with_capacity(layers.len()); let mut tier_k = Vec::with_capacity(layers.len()); + let mut downs = DownForms::NONE; for (l, spec) in layers.iter().enumerate() { // A dense block is outside the host run and has no row: no card // experts, which the resident-rows check below holds it to. A @@ -309,7 +328,13 @@ impl CardExperts { per.push(None); continue; } - LegStacks::of(w, layer.names())?.check(n_card, family.shape(n, ff, 1))?; + let stacks = LegStacks::of_arms(w, layer.names(), arms, STACK_ROLES)?; + stacks.check(n_card, family.shape(n, ff, 1))?; + let (_, act) = kernels::down(stacks.down_ty()).ok_or(GpuError::state( + what, + "the selected card down's activation form", + ))?; + downs = downs.with(act); per.push(Some(layer)); } Ok(CardExperts { @@ -317,14 +342,16 @@ impl CardExperts { k: CardKernels { leg: CardLeg::load(gpu)?, places: CardSlotKernels::load(gpu.context())?, + arms, }, layers: per, tier: tier_k, ff, n, - step: CardBufs::new(gpu.stream(), &family, n, ff, 1)?, + downs, + step: CardBufs::new(gpu.stream(), &family, n, ff, 1, downs)?, pass: (rows > 1) - .then(|| CardBufs::new(gpu.stream(), &family, n, ff, rows)) + .then(|| CardBufs::new(gpu.stream(), &family, n, ff, rows, downs)) .transpose()?, }) } @@ -410,17 +437,24 @@ impl CardExperts { pub fn batch(&self, gpu: &Gpu, cap: usize) -> Result { let stream = gpu.stream(); Ok(CardBatch { - bufs: CardBufs::new(stream, &self.family, self.n, self.ff, cap)?, + bufs: CardBufs::new(stream, &self.family, self.n, self.ff, cap, self.downs)?, sel: DeviceBuffer::zeroed(stream, cap * self.family.n_used)?, }) } /// Device bytes of [`CardExperts::batch`]'s buffers for `cap` tokens - /// ([`CardBatch::bytes`]'s formula): the card leg's ([`leg_bytes`]), the + /// ([`CardBatch::bytes`]'s formula): the card leg's, the /// card sum and the sum plus the base, and the places. #[must_use] pub fn batch_bytes(&self, cap: usize) -> usize { - leg_bytes(self.family.shape(self.n, self.ff, cap)) + let leg = kernels::leg_bytes( + cap as u64, + self.family.n_used as u64, + self.n as u64, + self.ff as u64, + self.downs, + ); + usize::try_from(leg).expect("the card leg's bytes fit a usize") + 2 * size_of::() * cap * self.n + size_of::() * cap * self.family.n_used } @@ -550,7 +584,7 @@ impl CardKernels { io: &Io<'_>, sel: &DeviceBuffer, ) -> Result<(), GpuError> { - let stacks = LegStacks::of(w, layer.names())?; + let stacks = LegStacks::of_arms(w, layer.names(), self.arms, STACK_ROLES)?; let run = LegRun { layer: l, stacks: &stacks, @@ -573,3 +607,325 @@ impl CardKernels { .enqueue_add(stream, &bufs.acc, io.x1, io.m * n, &mut bufs.pre) } } + +#[cfg(test)] +mod tests { + use super::*; + use gguf::GgmlType; + use gguf::write::{Layout, TensorDecl, Writer}; + use model::arch::models::{self, Mixer, Moe, Residual, Router, RouterNorm, Score}; + + use crate::card_leg::tests::stack_bytes; + use crate::host::leg_tier::LegTier; + use crate::host::tier::{TierBlock, TierExperts, TierInput, TierIo, TierSet}; + use crate::hybrid::HOST; + + const N: usize = 256; + const FF: usize = 256; + const USED: usize = 8; + const EXPERTS: usize = 16; + const CARD: usize = 4; + const CAP: usize = 3; + const FAMILY: CardFamily = CardFamily { + what: "mixed card wrapper", + n_expert: EXPERTS, + n_used: USED, + stacks: model::arch::names::routed_stacks, + program: Program::Qwen35Body, + }; + + fn layers(n: usize) -> Vec { + vec![ + LayerSpec { + mixer: Mixer::None, + ffn: Ffn::Moe(Moe { + experts: EXPERTS as u32, + top_k: USED as u32, + expert_ff: FF as u32, + latent: None, + act: models::Act::SwiGlu { limit: None }, + router: Router { + score: Score::Sigmoid, + bias: false, + norm: RouterNorm::SelectedSum, + scale: 1.0, + hash: false, + }, + shared: None, + }), + residual: Residual::Plain, + extras: Vec::new(), + }; + n + ] + } + + fn weights(gpu: &Gpu, triples: &[[GgmlType; 3]]) -> Weights { + let mut decls = Vec::new(); + let mut data = Vec::new(); + for (l, triple) in triples.iter().enumerate() { + for (part, (name, &ty)) in (FAMILY.stacks)(l).into_iter().zip(triple).enumerate() { + let (rows, k) = if part == 2 { (N, FF) } else { (FF, N) }; + let bytes = stack_bytes(ty, CARD * rows, k, 7 + (3 * l + part) as u64); + decls.push(TensorDecl { + name, + dims: vec![k as u64, rows as u64, CARD as u64], + type_id: ty.as_u32(), + nbytes: bytes.len() as u64, + }); + data.push(bytes); + } + } + static CALLS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + let call = CALLS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!( + "bloomery-mixed-wrappers-{}-{call}.gguf", + std::process::id() + )); + let layout = Layout::new(&[], decls.clone()).expect("a layout"); + let file = std::io::BufWriter::new(std::fs::File::create(&path).expect("a file")); + let mut writer = Writer::new(file, layout).expect("a writer"); + for (decl, bytes) in decls.iter().zip(data) { + writer.tensor(&decl.name, &bytes).expect("a tensor"); + } + writer.finish().expect("a finished file"); + let file = gguf::Split::open(&path).expect("the file"); + let weights = Weights::load_where(gpu.stream(), &file, |_| true).expect("resident stacks"); + std::fs::remove_file(path).expect("remove the fixture"); + weights + } + + fn assert_bits(got: &[f32], want: &[f32]) { + assert_eq!(got.len(), want.len()); + for (i, (&g, &w)) in got.iter().zip(want).enumerate() { + assert!(g.is_finite() && w.is_finite(), "lane {i}: {g} vs {w}"); + assert_eq!(g.to_bits(), w.to_bits(), "lane {i}: {g} vs {w}"); + } + } + + /// Each real wrapper uses its resident layers' union, not a model-wide + /// quant: both one-form loads and mixed loads conserve bytes and execute + /// step, pass, batch and tier chunks as the same common leg. Host slots + /// stay untouched on the tier, including a chunk's short final run. + #[test] + #[ignore = "needs a CUDA device; `just gate-gpu-lib` runs it on the box"] + fn hw_mixed_card_and_tier_wrappers_conserve_forms_and_rows() { + use GgmlType::{IQ3_S, IQ4_XS, Q5_K, Q6_K, Q8_0}; + let gpu = Gpu::with_device(0).expect("a Gpu on device 0"); + let stream = gpu.stream(); + let leg = CardLeg::load(&gpu).expect("the reference leg"); + for triples in [ + vec![[IQ3_S, IQ3_S, Q5_K], [IQ4_XS; 3]], + vec![[Q5_K, Q5_K, Q8_0]], + vec![[Q5_K, Q5_K, Q6_K]], + vec![ + [IQ3_S, IQ3_S, Q5_K], + [Q5_K, Q5_K, Q8_0], + [Q5_K, Q5_K, Q6_K], + [IQ4_XS; 3], + ], + ] { + let w = weights(&gpu, &triples); + let specs = layers(triples.len()); + let map = SlotMap::prefix(0..triples.len(), EXPERTS, CARD).expect("a map"); + let mut card = CardExperts::new(&gpu, &w, FAMILY, &specs, &map, (N, FF), CAP) + .expect("the card wrapper"); + let mut batch = card.batch(&gpu, CAP).expect("the batch"); + assert_eq!(card.batch_bytes(CAP), batch.bytes()); + let forms = triples.iter().fold(DownForms::NONE, |forms, tys| { + forms.with(kernels::down(tys[2]).expect("a down").1) + }); + let counted = |cap| { + kernels::leg_bytes(cap as u64, USED as u64, N as u64, FF as u64, forms) as usize + }; + assert_eq!( + card.bytes(), + counted(1) + counted(CAP) + 2 * size_of::() * (1 + CAP) * N, + ); + assert_eq!( + batch.bytes(), + counted(CAP) + 2 * size_of::() * CAP * N + size_of::() * CAP * USED, + ); + let shape = FAMILY.shape(N, FF, 2); + let set = TierSet::new( + 0..triples.len(), + EXPERTS, + vec![(0..CARD as u32).collect(); triples.len()], + ) + .expect("the tier set"); + let mut tier = LegTier::new(&gpu, &specs, &set, &w, shape, FAMILY.program) + .expect("the tier wrapper"); + assert_eq!(tier.block_bytes(), counted(shape.cap)); + let host_x: Vec = (0..CAP * N).map(|i| (i % 71) as f32 / 71.0 - 0.5).collect(); + let x = DeviceBuffer::from_host(stream, &host_x).expect("the rows"); + let base: Vec = (0..CAP * N).map(|i| (i % 13) as f32 * 0.125).collect(); + let x1 = DeviceBuffer::from_host(stream, &base).expect("the base"); + let route = + DeviceBuffer::from_host(stream, &vec![0.125f32; CAP * USED]).expect("weights"); + let host_sel: Vec = (0..CAP * USED) + .map(|i| { + if i % USED == USED - 1 { + HOST + } else { + (i % CARD) as u32 + } + }) + .collect(); + let sel = DeviceBuffer::from_host(stream, &host_sel).expect("the places"); + let ids = DeviceBuffer::from_host( + stream, + &host_sel + .iter() + .map(|&s| if s == HOST { (EXPERTS - 1) as u32 } else { s }) + .collect::>(), + ) + .expect("the ids"); + let places = DeviceTensor::upload(stream, &map.stage_view(), triples.len(), EXPERTS) + .expect("the map's card view"); + let act = crate::Q8Act::with_tier_cols(stream, CAP, N).expect("the tier block's form"); + for (l, tys) in triples.iter().enumerate() { + let names = (FAMILY.stacks)(l); + let stacks = LegStacks::of_arms( + &w, + names.each_ref().map(String::as_str), + accepts::leg_arms(FAMILY.program).expect("the selected arms"), + STACK_ROLES, + ) + .expect("the reference stacks"); + let mut reference = LegBufs::for_down( + stream, + FAMILY.shape(N, FF, CAP), + kernels::down(tys[2]).expect("a down").1, + ) + .expect("the reference buffers"); + let mut acc = DeviceBuffer::zeroed(stream, CAP * N).expect("the reference sum"); + for m in [1, CAP] { + let io = Io { + normed: &x, + weights: &route, + x1: &x1, + m, + }; + let rule = Act::SwigluClamp { limit: 0.0 }; + leg.enqueue( + &gpu, + &mut reference, + &LegRun { + layer: l, + stacks: &stacks, + normed: &x, + m, + sel: &sel, + n_card: CARD, + rule, + }, + Some(LegSum { + w: &route, + acc: &mut acc, + }), + ) + .expect("the reference"); + let sums = acc.to_host_vec(stream).expect("the reference sums"); + let want: Vec = sums[..m * N] + .iter() + .zip(&base) + .map(|(&a, &b)| a + b) + .collect(); + let got = if m == 1 { + card.enqueue_step(&gpu, &w, l, rule, &io, &sel) + .expect("step"); + card.pre() + } else { + card.enqueue_pass(&gpu, &w, l, rule, &io, &sel) + .expect("pass"); + card.pass_pre().expect("pass output") + }; + assert_bits(&got.to_host_vec(stream).expect("output")[..m * N], &want); + card.enqueue_rows( + &gpu, + &w, + &mut batch, + l, + rule, + &io, + (&ids, &places, l * EXPERTS), + ) + .expect("batch"); + assert_bits( + &batch.pre().to_host_vec(stream).expect("batch output")[..m * N], + &want, + ); + let mut down = + DeviceBuffer::from_host(stream, &vec![1.0e30f32; CAP * USED * N]) + .expect("the tier output"); + if m == 1 { + tier.enqueue_layer( + &gpu, + &w, + l, + TierIo { + act: TierInput::F32(&x), + sel: &sel, + rows: &mut down, + cols: m, + }, + ) + .expect("tier step"); + } else { + tier.enqueue_block( + &gpu, + &w, + l, + TierBlock { + x: &x, + act: &act, + sel: &sel, + cols: m, + down: &mut down, + }, + ) + .expect("tier chunks"); + } + let got = down.to_host_vec(stream).expect("tier output"); + let want = reference + .down() + .to_host_vec(stream) + .expect("reference rows"); + for (s, &place) in host_sel[..m * USED].iter().enumerate() { + if place == HOST { + assert!(got[s * N..(s + 1) * N].iter().all(|&v| v == 1.0e30)); + } else { + assert_bits(&got[s * N..(s + 1) * N], &want[s * N..(s + 1) * N]); + } + } + } + } + assert!(gpu.take_fault().expect("the fault word").is_none()); + } + for (triple, program) in [ + ([IQ3_S, Q5_K, Q8_0], Program::Qwen35Body), + ([Q5_K, Q5_K, Q6_K], Program::Solar2Body), + ] { + let w = weights(&gpu, &[triple]); + let specs = layers(1); + let map = SlotMap::prefix(0..1, EXPERTS, CARD).expect("a map"); + let family = CardFamily { program, ..FAMILY }; + assert!( + matches!( + CardExperts::new(&gpu, &w, family, &specs, &map, (N, FF), CAP), + Err(GpuError::Tensor { .. }), + ), + "{program:?}: {triple:?}" + ); + let set = + TierSet::new(0..1, EXPERTS, vec![(0..CARD as u32).collect()]).expect("a tier set"); + assert!( + matches!( + LegTier::new(&gpu, &specs, &set, &w, family.shape(N, FF, 2), program), + Err(GpuError::Tensor { .. }), + ), + "{program:?}: {triple:?}" + ); + } + } +} diff --git a/crates/gpu/src/card_leg.rs b/crates/gpu/src/card_leg.rs index 54242a724..91c0f270a 100644 --- a/crates/gpu/src/card_leg.rs +++ b/crates/gpu/src/card_leg.rs @@ -555,8 +555,8 @@ impl<'a> LegStacks<'a> { w, names, LegArms { - gate_up: kernels::card_gate_up_runs, - down: kernels::card_down_runs, + gate_up: accepts::planes_leg_gate_up, + down: accepts::planes_leg_down, }, (GATE_UP, DOWN), ) @@ -571,8 +571,8 @@ impl<'a> LegStacks<'a> { w, names, LegArms { - gate_up: kernels::card_gate_up_runs, - down: kernels::rank_down_runs, + gate_up: accepts::planes_leg_gate_up, + down: |ty| kernels::rank_down_runs(ty) || kernels::card_planes_runs(ty), }, (GATE_UP, DOWN), ) @@ -589,8 +589,8 @@ impl<'a> LegStacks<'a> { w, names, LegArms { - gate_up: kernels::card_gate_up_runs, - down: accepts::resident_down_runs, + gate_up: accepts::planes_leg_gate_up, + down: |ty| accepts::resident_down_runs(ty) || kernels::card_planes_runs(ty), }, (GATE_UP, DOWN), ) @@ -610,9 +610,9 @@ impl<'a> LegStacks<'a> { ) -> Result, GpuError> { let [gate, up, down] = names; let (gate, up, down) = ( - LegStack::of(w, gate, arms.gate_up, kernels::card_planes_runs, roles.0)?, - LegStack::of(w, up, arms.gate_up, kernels::card_planes_runs, roles.0)?, - LegStack::of(w, down, arms.down, kernels::card_planes_runs, roles.1)?, + LegStack::of(w, gate, arms.gate_up, arms.gate_up, roles.0)?, + LegStack::of(w, up, arms.gate_up, arms.gate_up, roles.0)?, + LegStack::of(w, down, arms.down, arms.down, roles.1)?, ); if up.ty != gate.ty { return Err(GpuError::Tensor { diff --git a/crates/gpu/src/host/batch.rs b/crates/gpu/src/host/batch.rs index 5a69a4707..29dc6c687 100644 --- a/crates/gpu/src/host/batch.rs +++ b/crates/gpu/src/host/batch.rs @@ -29,7 +29,7 @@ use crate::graph::{MappedHost, cu}; use crate::tensor::WindowMut; use crate::{Gpu, GpuError, Q8Act}; use cuda_core::{CudaContext, CudaEvent, CudaStream, DeviceBuffer, PinnedHostBuffer, sys}; -use model::placement::Machine; +use model::placement::Plan; use model::placement::workstation::{TIER_BATCH_HOST_RESERVE, TIER_BATCH_RESERVE}; use model::{Tensor2, Tensor2View}; use std::mem::size_of; @@ -1079,18 +1079,19 @@ impl HostTier { Ok(()) } - /// The expert tiers' prompt-batch bytes against the reserves `machine`'s + /// The expert tiers' prompt-batch bytes against the selected reserves the /// plan carries for them: on each tier's card — tier `t`'s is index - /// `cards[t]` of [`Machine::all_cards`] — its batch staging and the + /// `cards[t]` of [`model::placement::Machine::all_cards`] — its batch staging and the /// architecture's block scratch ([`TierCard::block_bytes`]) against that - /// card's [`TIER_BATCH_RESERVE`] row; on the host, the tier legs' rows and + /// card's selected [`TIER_BATCH_RESERVE`] row in `plan`; on the host, the tier legs' rows and /// places against the host's [`TIER_BATCH_HOST_RESERVE`] row. A card list /// that is not one card a tier, a missing row, a row named twice, or a /// difference is refused by name, naming the reserve. Load-time only, once /// [`HostTier::prepare_batch`] has made the legs. Step-only tiers make /// no batch leg ([`HostTier::prepare_tier_batch`]) and none is checked. - pub fn check_tier_reserves(&self, machine: &Machine, cards: &[usize]) -> Result<(), GpuError> { + pub fn check_tier_reserves(&self, plan: &Plan<'_>, cards: &[usize]) -> Result<(), GpuError> { const WHAT: &str = "HostTier::check_tier_reserves"; + let machine = plan.machine; if self.tiers.iter().any(|t| !t.supports_blocks()) { return Ok(()); } @@ -1124,6 +1125,7 @@ impl HostTier { &card.reserves, TIER_BATCH_RESERVE, port.tier_device_bytes(i) + t.block_bytes()?, + Some(index), )); } rows.push(( @@ -1131,15 +1133,27 @@ impl HostTier { &machine.host().reserves, TIER_BATCH_HOST_RESERVE, port.tier_host_bytes(), + None, )); - for (on, reserves, name, got) in rows { + for (on, reserves, name, got, card) in rows { let want = match reserves .iter() .filter(|(n, _)| n == name) .collect::>() .as_slice() { - [(_, b)] => *b, + [(_, b)] => match card { + Some(c) => plan + .card_reserve(c, name) + .map_err(|e| GpuError::plan(WHAT, e))? + .ok_or_else(|| { + GpuError::shape( + WHAT, + format!("the plan's selected reserve {name:?} on {on} is invalid"), + ) + })?, + None => *b, + }, named => { return Err(GpuError::shape( WHAT, diff --git a/crates/gpu/src/host/leg_tier.rs b/crates/gpu/src/host/leg_tier.rs index 5620547a4..a77459fa0 100644 --- a/crates/gpu/src/host/leg_tier.rs +++ b/crates/gpu/src/host/leg_tier.rs @@ -8,8 +8,8 @@ //! //! The tier's layer is the card leg ([`CardLeg::enqueue_rows`]) over the //! tier's stacks: the q8_1 of the activation, the gate·up `_sel` with the -//! routed clamped SwiGLU, the q8_1 of the tier slots' columns and the down -//! `_sel` into the tier's rows; over a prompt batch's block, the same run +//! routed clamped SwiGLU, the selected down form of the tier slots' columns +//! and the down `_sel` into the tier's rows; over a prompt batch's block, the same run //! chunk by chunk of `shape.cap` tokens. Each slot's down output is the one //! the stage card would write for it, so the card sum over both cards' slots //! in slot order is the one-card sum over their union. The row's tier image @@ -26,6 +26,8 @@ use model::ModelError; use model::arch::models::{Ffn, LayerSpec}; use model::arch::names::routed_stacks; use model::moe::HostLayer; +use model::placement::accepts::{self, LegArms, Program}; +use model::placement::kernels::{self, DownForms}; use model::placement::{ModelTensors, Plan}; use model::r8file::R8Source; @@ -46,6 +48,12 @@ const WHAT: &str = "LegTier"; /// What the stage card's side of the tier layers names in its errors. const SIDE: &str = "TierSide"; +/// The selected program's launches admit the same stacks on both cards. +const STACK_ROLES: (&str, &str) = ( + "a resident routed gate or up stack the selected tier gate·up arm runs", + "a resident routed down stack the selected tier down arm runs", +); + /// The tier the stage card's tier entries serve: they hand one tier its /// image and join one tier's rows, so a load holds one tier card /// ([`crate::host::refuse_tier_count`]). @@ -173,6 +181,7 @@ pub struct LegTier { /// Per layer of `layers`, `None` off the tier. cfg: Vec>, shape: LegShape, + arms: LegArms, } impl LegTier { @@ -188,7 +197,13 @@ impl LegTier { set: &TierSet, w: &Weights, shape: LegShape, + program: Program, ) -> Result { + let arms = accepts::leg_arms(program).ok_or(GpuError::state( + WHAT, + "the selected program's card leg arms", + ))?; + let mut downs = DownForms::NONE; let run = set.layers(); let mut cfg = Vec::with_capacity(run.len()); for l in run.clone() { @@ -216,15 +231,22 @@ impl LegTier { limit: m.act.swiglu_limit(), names, }; - LegStacks::of(w, layer.names())?.check(k, shape)?; + let stacks = LegStacks::of_arms(w, layer.names(), arms, STACK_ROLES)?; + stacks.check(k, shape)?; + let (_, act) = kernels::down(stacks.down_ty()).ok_or(GpuError::state( + WHAT, + "the selected tier down's activation form", + ))?; + downs = downs.with(act); cfg.push(Some(layer)); } Ok(LegTier { leg: CardLeg::load(gpu)?, - bufs: LegBufs::new(gpu.stream(), shape)?, + bufs: LegBufs::for_downs(gpu.stream(), shape, downs)?, layers: run, cfg, shape, + arms, }) } } @@ -270,6 +292,7 @@ impl TierExperts for LegTier { layers, cfg, shape, + arms, } = self; let LegShape { cap, @@ -278,7 +301,7 @@ impl TierExperts for LegTier { .. } = *shape; let tl = layer_of(layers, cfg, layer, W)?; - let stacks = LegStacks::of(weights, tl.names())?; + let stacks = LegStacks::of_arms(weights, tl.names(), *arms, STACK_ROLES)?; for c0 in (0..io.cols).step_by(cap) { let cn = cap.min(io.cols - c0); let slots = cn * slots_per_col; @@ -321,7 +344,7 @@ impl TierExperts for LegTier { .to_string(), }); }; - let stacks = LegStacks::of(weights, tl.names())?; + let stacks = LegStacks::of_arms(weights, tl.names(), self.arms, STACK_ROLES)?; let run = LegRun { layer, stacks: &stacks, @@ -432,6 +455,8 @@ pub struct LegWidths { pub n_embd: usize, /// A routed expert's width. pub ff: usize, + /// The program whose card leg runs these experts. + pub program: Program, } impl LegWidths { @@ -523,13 +548,15 @@ fn node_experts( open: &NodeOpen, ) -> Result, GpuError> { let inputs = node_inputs::(file, open)?; - let shape = F::widths(&inputs).shape(F::CHUNK); + let widths = F::widths(&inputs); + let shape = widths.shape(F::CHUNK); Ok(Box::new(LegTier::new( gpu, F::layers(&inputs), &open.set, w, shape, + widths.program, )?)) } @@ -603,5 +630,5 @@ pub fn hang_tiers( let planned: Vec = hang.tiers.iter().map(|t| t.card).collect(); hybrid.attach_tiers(cards, gpu)?; hybrid.prepare_batch(gpu.context(), hang.cols)?; - hybrid.check_tier_reserves(plan.machine, &planned) + hybrid.check_tier_reserves(plan, &planned) } diff --git a/crates/gpu/src/host/node.rs b/crates/gpu/src/host/node.rs index ed5fb0bc4..b15500b2f 100644 --- a/crates/gpu/src/host/node.rs +++ b/crates/gpu/src/host/node.rs @@ -320,13 +320,13 @@ impl NodeOpen { .find(|d| self.device.is_some_and(|id| id.uuid == d.uuid)) .ok_or_else(|| GpuError::shape(WHAT, "the projected tier is absent from the census"))?; let machine = workstation::plan_on(spec_of_device(device), self.layers); - let reserve = plan - .machine - .card(self.card) - .ok_or_else(|| { - GpuError::shape(WHAT, "the projected tier card is absent from the plan") - })? - .set_aside_bytes() + let totals = plan.cards.get(self.card).ok_or_else(|| { + GpuError::shape(WHAT, "the projected tier card is absent from the plan") + })?; + let reserve = totals + .context_bytes + .checked_add(totals.scratch_bytes) + .and_then(|bytes| bytes.checked_add(totals.reserve_bytes)) .ok_or_else(|| GpuError::shape(WHAT, "the projected tier reserve overflows"))?; let mut projected = model::placement::nodes::routed_host( plan.model, diff --git a/crates/model/src/arch/deepseek41/kv.rs b/crates/model/src/arch/deepseek41/kv.rs index 97506d933..c6eb5d744 100644 --- a/crates/model/src/arch/deepseek41/kv.rs +++ b/crates/model/src/arch/deepseek41/kv.rs @@ -262,6 +262,7 @@ mod tests { let machine = Machine { cards: vec![Card { name: "pool".into(), + tier_leg: None, device: None, usable_bytes: pool, context_bytes: 0, diff --git a/crates/model/src/arch/glm5next/place.rs b/crates/model/src/arch/glm5next/place.rs index 477567fbd..20ef61907 100644 --- a/crates/model/src/arch/glm5next/place.rs +++ b/crates/model/src/arch/glm5next/place.rs @@ -137,15 +137,6 @@ fn card_leg_forms(model: &ModelTensors) -> DownForms { .fold(DownForms::NONE, |forms, (_, act)| forms.with(act)) } -/// The first routed layer of `model` the card experts run that the expert -/// tier card's leg does not, with its stacks' types: that leg reads the -/// common leg's types in every part ([`kernels::card_runs`]: the file's blocks -/// over the 128-value form, the one its buffers and the plans' reserve hold), -/// so a plan that hangs a tier card is refused it ([`PlaceError::TierLeg`]). -fn off_tier_layer(model: &ModelTensors) -> Option<(usize, [GgmlType; 3])> { - carded_layers(model).find(|(_, tys)| !tys.iter().all(|&ty| kernels::card_runs(ty))) -} - /// Tokens a chunk of the tier's prompt-batch service runs at most: the /// stage card's chunk of the card experts' launches, which the body binds /// to its own constant. @@ -156,14 +147,15 @@ pub const TIER_CHUNK: u64 = 8; /// path's scratch ([`workstation::tier_chunk_scratch_bytes`]) and the host's /// rows and places, which the plan reserves on the tier card and the host /// and the load checks its allocations against. -#[must_use] -pub fn tier_batch(hp: &Hparams) -> TierBatchBytes { +pub fn tier_batch(inputs: &PlanInputs) -> Result { + let hp = &inputs.hp; workstation::tier_leg_batch_bytes( hp.n_embd as u64, hp.expert_ff as u64, hp.n_used as u64, TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64, + crate::placement::accepts::leg_down_forms(Program::Glm5nextBody, &inputs.model), ) } @@ -419,16 +411,6 @@ pub enum PlaceError { /// A plan of no resident sequence: a load serves at least one. #[error("a plan of {slots} resident sequences: a load serves at least one")] Slots { slots: usize }, - /// A machine that hangs an expert tier card beside a file with a card - /// layer the tier's leg does not run (NVFP4's planes, a down over the - /// 32-value form): the tier card's buffers and reserve hold the file's - /// blocks over the 128-value form alone. - #[error( - "layer {layer}: routed stacks {tys:?} run on the stage card's leg, and the expert tier \ - card's leg reads the file's blocks over 128-value q8_1 columns alone; plan it without \ - the tier card" - )] - TierLeg { layer: usize, tys: [GgmlType; 3] }, } impl PlanRefusal for PlaceError { @@ -654,7 +636,6 @@ impl PlanInputs { if slots == 0 { return Err(PlaceError::Slots { slots }); } - self.refuse_off_tier(machine)?; let kv = self.kv.with_lanes(lanes); let terms = self.seq_terms(&kv, None); let n = slots as u64; @@ -700,18 +681,6 @@ impl PlanInputs { } Ok(plan) } - - /// Refused by name ([`PlaceError::TierLeg`]) when `machine` hangs an - /// expert tier card and a card layer of the file is one the tier's leg - /// does not run. - fn refuse_off_tier(&self, machine: &Machine) -> Result<(), PlaceError> { - match off_tier_layer(&self.model) { - Some((layer, tys)) if !machine.tiers.is_empty() => { - Err(PlaceError::TierLeg { layer, tys }) - } - _ => Ok(()), - } - } } /// Card bytes the NextN walk's arena may hold beside the plan's tensors @@ -832,6 +801,7 @@ fn nextn_machine() -> Machine { Machine { cards: vec![Card { name: "nextn".to_string(), + tier_leg: None, device: None, usable_bytes: u64::MAX, context_bytes: 0, @@ -1015,7 +985,6 @@ impl PlanInputs { if slots == 0 { return Err(PlaceError::Slots { slots }); } - self.refuse_off_tier(machine)?; let kv = self.kv.with_lanes(KdaLanes::Two); let terms = self.seq_terms(&kv, Some(nextn)); let n = slots as u64; @@ -1591,6 +1560,7 @@ mod tests { Machine { cards: vec![Card { name: "card".to_string(), + tier_leg: None, device: None, usable_bytes: 1 << 40, context_bytes: 1 << 20, @@ -2155,44 +2125,74 @@ mod tests { } } - /// A machine that hangs an expert tier card is refused, by layer - /// and types, a file with a card layer the tier's leg does not run - /// (an NVFP4 layer: its planes and 32-value form); without the tier - /// card the same file plans, the NVFP4 layer's experts on the stage - /// card, and a K-quant file is not refused with it - /// ([`node_ranges_use_the_common_loaded_assignment`]). Mutants: the - /// refusal dropped, which leaves the tier's load to refuse the - /// layer's stacks; the body's plan rule left on the common leg's, which - /// keeps the NVFP4 layer on the host no kernel there reads. + /// The tier reserves precisely the selected layers' down forms, + /// including a mixed 128/32-value load; a gate/up mismatch remains + /// host-only and cannot add a form. #[test] - fn a_tier_card_is_refused_a_layer_its_leg_does_not_run() { + fn tier_scratch_follows_selected_mixed_layers() { + use super::super::TIER_CHUNK; + use crate::placement::kernels::{self, DownAct, DownForms}; + use GgmlType::{IQ3_S, NVFP4, Q5_K, Q8_0}; let mut inputs = routed(64 << 30); - for t in inputs + inputs .model .tensors - .iter_mut() - .filter(|t| t.role == Role::RoutedExperts && t.layer == Some(1)) - { - t.ty = GgmlType::NVFP4; - t.file_bytes = t.dims.iter().product::() / 64 * 36; - } - let plain = unified(64 << 30); - let mut machine = plain.clone(); - let mut tier = machine.cards[0].clone(); - tier.name = "remote".into(); - tier.head = false; - tier.layers = 0..0; - machine.tiers.push(tier); - match inputs.plan(&machine, CTX, &PlanLevers::default()) { - Err(PlaceError::TierLeg { layer, tys }) => { - assert_eq!((layer, tys), (1, [GgmlType::NVFP4; 3])); + .retain(|t| t.role != Role::RoutedExperts || t.layer.is_some_and(|l| l < 2)); + let set = |inputs: &mut PlanInputs, l, tys: [GgmlType; 3]| { + for t in inputs + .model + .tensors + .iter_mut() + .filter(|t| t.role == Role::RoutedExperts && t.layer == Some(l)) + { + let part = match crate::placement::ExpertPart::of_name(&t.name) { + Some(crate::placement::ExpertPart::Gate) => 0, + Some(crate::placement::ExpertPart::Up) => 1, + Some(crate::placement::ExpertPart::Down) => 2, + None => unreachable!(), + }; + t.ty = tys[part]; + } + }; + for (layers, forms) in [ + ( + [[IQ3_S, IQ3_S, Q5_K], [Q5_K, Q5_K, Q5_K]], + DownForms::of(DownAct::Q8Act), + ), + ( + [[Q5_K, Q5_K, Q8_0], [Q5_K, Q5_K, Q8_0]], + DownForms::of(DownAct::Q8Blocks32), + ), + ( + [[IQ3_S, IQ3_S, Q5_K], [NVFP4; 3]], + DownForms::of(DownAct::Q8Act).with(DownAct::Q8Blocks32), + ), + ( + [[IQ3_S, IQ3_S, Q5_K], [Q5_K, Q5_K, Q8_0]], + DownForms::of(DownAct::Q8Act).with(DownAct::Q8Blocks32), + ), + ( + [[IQ3_S, IQ3_S, Q5_K], [IQ3_S, Q5_K, Q8_0]], + DownForms::of(DownAct::Q8Act), + ), + ] { + for (l, tys) in layers.into_iter().enumerate() { + set(&mut inputs, l, tys); } - other => panic!("{:?}", other.map(|p| p.n_l.clone())), + let bytes = super::super::tier_batch(&inputs).expect("tier bytes"); + let hp = &inputs.hp; + assert_eq!( + bytes.scratch, + kernels::leg_bytes( + TIER_CHUNK, + hp.n_used as u64, + hp.n_embd as u64, + hp.expert_ff as u64, + forms, + ), + "{layers:?}", + ); } - let p = inputs - .plan(&plain, CTX, &PlanLevers::default()) - .expect("the plan without the tier card"); - assert!(p.n_l[1] > 0, "the NVFP4 layer on the card: {:?}", p.n_l); } #[test] diff --git a/crates/model/src/arch/qwen35moe/place.rs b/crates/model/src/arch/qwen35moe/place.rs index 2358730f1..6bd64a8f1 100644 --- a/crates/model/src/arch/qwen35moe/place.rs +++ b/crates/model/src/arch/qwen35moe/place.rs @@ -1507,6 +1507,7 @@ const fn tier_batch_of(n_embd: u64, n_used: u64, ubatch: u64) -> TierBatchBytes staging: tier_batch_staging_bytes(n_embd, n_used, cols), scratch: tier_route_scratch_bytes(cols), host: tier_batch_host_bytes(n_embd, n_used, cols), + leg: None, } } @@ -2332,6 +2333,7 @@ fn draft_machine() -> Machine { Machine { cards: vec![Card { name: "mtp draft".to_string(), + tier_leg: None, device: None, usable_bytes: u64::MAX, context_bytes: 0, @@ -2802,6 +2804,7 @@ mod tests { Machine { cards: vec![Card { name: "card".to_string(), + tier_leg: None, device: None, usable_bytes: usable, context_bytes: 0, @@ -3567,6 +3570,7 @@ mod tests { Machine { cards: vec![Card { name: "card".to_string(), + tier_leg: None, device: None, usable_bytes: 1 << 40, context_bytes: 0, @@ -4416,6 +4420,7 @@ mod tests { staging: 496_730_112, scratch: 348_980_248, host: 839_188_480, + leg: None, } ); assert_eq!( @@ -4424,6 +4429,7 @@ mod tests { staging: 62_091_264, scratch: 87_233_816, host: 104_898_560, + leg: None, } ); } diff --git a/crates/model/src/arch/solar2/place.rs b/crates/model/src/arch/solar2/place.rs index 2b186aeb2..c345ea2f8 100644 --- a/crates/model/src/arch/solar2/place.rs +++ b/crates/model/src/arch/solar2/place.rs @@ -13,9 +13,10 @@ use super::hparams::Hparams; use super::{roles, spec}; use crate::arch::chat_of; use crate::arch::place_hybrid; +use crate::placement::accepts::{Program, leg_down_forms}; use crate::placement::kernels::q8_act_bytes; use crate::placement::workstation::TierBatchBytes; -use crate::placement::{PlacementError, workstation}; +use crate::placement::{ModelTensors, PlacementError, workstation}; pub use place_hybrid::{KvLayout, PlaceError}; @@ -43,18 +44,19 @@ pub const GEMM32_STEP: usize = 64; /// the stage card's card experts, which the body binds to its own constant. pub const TIER_CHUNK: u64 = 8; -/// The expert tier's prompt-batch bytes ([`TierBatchBytes`]) for a file of hyperparameters `hp`: +/// The expert tier's prompt-batch bytes ([`TierBatchBytes`]) for a described file: /// the tier card's staging, the leg's chunk scratch and the host's rows and places, for blocks of /// the host union's columns. The plan reserves them on the tier card and the host, and the load /// checks its allocations against them. -#[must_use] -pub fn tier_batch(hp: &Hparams) -> TierBatchBytes { +pub fn tier_batch(inputs: &PlanInputs) -> Result { + let hp = &inputs.hp; workstation::tier_leg_batch_bytes( hp.n_embd as u64, hp.experts.ff as u64, hp.experts.n_used as u64, TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64, + leg_down_forms(Program::Solar2Body, &inputs.model), ) } @@ -128,8 +130,12 @@ const fn acts_bytes(k: usize) -> usize { /// The wide forms are a bound: both forms at each width that allows them, where the load makes /// the ones its sites' types read. The units a prefill group past one adds are made out of the /// card's margin. -#[must_use] -pub fn prompt_reserve_bytes(hp: &Hparams, spec: &ModelSpec, rows: usize) -> u64 { +pub fn prompt_reserve_bytes( + hp: &Hparams, + spec: &ModelSpec, + model: &ModelTensors, + rows: usize, +) -> Result { let (embd, r) = (hp.n_embd, rows); let (mut heads, mut kv_heads, mut head) = (hp.n_head, 0, hp.head_dim); let (mut k_heads, mut v_heads, mut d) = (0, 0, 0); @@ -163,13 +169,21 @@ pub fn prompt_reserve_bytes(hp: &Hparams, spec: &ModelSpec, rows: usize) -> u64 let wide: usize = others.iter().map(|&k| wide_bytes(r, k)).sum::() + 2 * wide_bytes(r, embd) + wide_bytes(r, shared_ff); - let card_leg = workstation::tier_chunk_scratch_bytes( - embd as u64, - experts.ff as u64, - experts.n_used as u64, - r as u64, - ) as usize; - (4 * f32s + acts + wide + 2 * route_bytes(r) + card_leg) as u64 + let downs = leg_down_forms(Program::Solar2Body, model); + let card_leg = workstation::TierLegScratch { + n_embd: embd as u64, + ff: experts.ff as u64, + n_used: experts.n_used as u64, + chunk: r as u64, + downs, + } + .bytes(downs)?; + ((4 * f32s + acts + wide + 2 * route_bytes(r)) as u64) + .checked_add(card_leg) + .ok_or_else(|| PlacementError::Metadata { + key: "solar2 prompt reserve".into(), + detail: "bytes overflow u64".into(), + }) } impl PlanInputs { @@ -186,7 +200,7 @@ impl PlanInputs { let model = roles::classify(split, &hp)?; // No parser is declared: see `spec`. let spec = spec::spec_of(&hp, chat_of(split, None, None)?)?; - let reserve = prompt_reserve_bytes(&hp, &spec, PROMPT_ROWS); + let reserve = prompt_reserve_bytes(&hp, &spec, &model, PROMPT_ROWS)?; let tier = tier_places_bytes(&hp, PROMPT_ROWS); Ok(PlanInputs::new(ENGINE, hp, model, spec)? .with_prompt_reserve(reserve) @@ -288,7 +302,7 @@ mod tests { #[test] fn a_tier_card_is_planned_with_the_batch_places_beside_the_reserve() { let inputs = described("so2-place-tier"); - let tiered = workstation::plan_bp(LAYERS, None, tier_batch(&inputs.hp)); + let tiered = workstation::plan_bp(LAYERS, None, tier_batch(&inputs).expect("tier bytes")); let plain = workstation::plan_gate(LAYERS); let places = 12 * 512 * inputs.hp.experts.n_used as u64; assert_eq!(tier_places_bytes(&inputs.hp, PROMPT_ROWS), places); @@ -326,11 +340,108 @@ mod tests { /// expert. Mutant: the reserve left out of the plan's call (the card plans, and its load stops /// at the allocation). #[test] - fn a_card_short_of_the_prompt_reserve_is_refused_by_name() { - let inputs = described("so2-place-reserve"); - let reserve = prompt_reserve_bytes(&inputs.hp, &inputs.spec, PROMPT_ROWS); - assert!(reserve > 0, "the synthetic header's batch has bytes"); - assert_eq!(inputs.prompt_reserve, reserve, "describe keeps the figure"); + fn a_host_only_mixture_keeps_the_exact_prompt_reserve_boundary() { + use crate::placement::{ExpertPart, Role}; + use gguf::GgmlType::{Q5_K, Q8_0}; + + let mut kv = keys(); + for (key, value) in &mut kv { + if matches!(*key, "embedding_length" | "expert_feed_forward_length") { + *value = V::U32(256); + } + } + let mut ts = tensors(); + for (name, dims) in &mut ts { + let tail = name.rsplit('.').nth(1).unwrap_or(""); + if matches!( + tail, + "ffn_gate_exps" + | "ffn_up_exps" + | "ffn_down_exps" + | "ffn_gate_shexp" + | "ffn_up_shexp" + | "ffn_down_shexp" + ) { + dims[0] = 256; + dims[1] = 256; + } else if tail == "attn_output" { + dims[1] = 256; + } else if matches!( + tail, + "token_embd" + | "output" + | "output_norm" + | "attn_norm" + | "ffn_norm" + | "attn_q" + | "attn_k" + | "attn_v" + | "attn_gate" + | "ssm_f_a" + | "ssm_g_a" + | "ssm_beta" + | "ffn_gate_inp" + ) { + dims[0] = 256; + } + } + let global = [ + ("tokenizer.ggml.pre", V::Str("solar-open")), + ("tokenizer.chat_template", V::Str("{{ messages }}")), + ]; + let path = header_shaped("so2-place-mixed-reserve", "solar-open2", &kv, &global, &ts); + let split = Split::open(&path).expect("the mixed fixture opens"); + let mut inputs = PlanInputs::describe(&split).expect("valid routed geometry"); + std::fs::remove_file(path).expect("remove the header fixture"); + for t in inputs + .model + .tensors + .iter_mut() + .filter(|t| t.role == Role::RoutedExperts) + { + t.ty = Q5_K; + t.file_bytes = t.dims.iter().product::() / t.ty.blck_size().unwrap() + * t.ty.type_size().unwrap(); + } + let admitted = prompt_reserve_bytes(&inputs.hp, &inputs.spec, &inputs.model, PROMPT_ROWS) + .expect("the admitted prompt reserve"); + let admitted_tier = tier_batch(&inputs).expect("the admitted tier reserve"); + let down = inputs + .model + .tensors + .iter_mut() + .find(|t| { + t.role == Role::RoutedExperts + && t.layer == Some(0) + && ExpertPart::of_name(&t.name) == Some(ExpertPart::Down) + }) + .expect("layer zero's down"); + down.ty = Q8_0; + down.file_bytes = down.dims.iter().product::() / down.ty.blck_size().unwrap() + * down.ty.type_size().unwrap(); + let reserve = prompt_reserve_bytes(&inputs.hp, &inputs.spec, &inputs.model, PROMPT_ROWS) + .expect("prompt reserve"); + assert_eq!( + reserve, admitted, + "the host-only Q8_0 down adds no card form" + ); + assert_eq!( + tier_batch(&inputs).expect("mixed tier reserve"), + admitted_tier + ); + inputs.prompt_reserve = reserve; + let ample = workstation::plan_gate(LAYERS); + let placed = inputs + .plan(&le, 64, &PlanLevers::default()) + .expect("the mixed plan"); + assert_eq!( + placed.n_l[0], 0, + "Q8_0 stays on the host under Solar's rule" + ); + assert!( + placed.n_l[1..].iter().all(|&n| n > 0), + "other layers remain card candidates" + ); let levers = PlanLevers::default(); let mut machine = workstation::plan_gate(LAYERS); machine.cards[0].usable_bytes = machine.cards[0].margin_bytes; @@ -367,10 +478,6 @@ mod tests { let plan = inputs .plan(&machine, 64, &levers) .expect("the card holds the terms and the reserve"); - assert_eq!( - plan.n_l.iter().sum::(), - 0, - "no room is left for an expert" - ); + assert!(plan.violations().is_empty()); } } diff --git a/crates/model/tests/placement.rs b/crates/model/tests/placement.rs index f3314b3c1..2f5d27b64 100644 --- a/crates/model/tests/placement.rs +++ b/crates/model/tests/placement.rs @@ -1368,6 +1368,7 @@ fn layered_model(layers: usize) -> ModelTensors { fn byte_card(name: &str, usable: u64, layers: Range) -> Card { Card { name: name.to_string(), + tier_leg: None, device: None, usable_bytes: usable, context_bytes: 0, diff --git a/crates/model/tests/qwen4exp_mtp_reserve.rs b/crates/model/tests/qwen4exp_mtp_reserve.rs index 8b976bbb0..8da38ef7a 100644 --- a/crates/model/tests/qwen4exp_mtp_reserve.rs +++ b/crates/model/tests/qwen4exp_mtp_reserve.rs @@ -295,6 +295,7 @@ fn machine(usable: u64) -> Machine { Machine { cards: vec![Card { name: "card".to_string(), + tier_leg: None, device: None, usable_bytes: usable, context_bytes: CONTEXT, diff --git a/crates/model/tests/solar_open2_meta.rs b/crates/model/tests/solar_open2_meta.rs index de58c80de..65465b0d7 100644 --- a/crates/model/tests/solar_open2_meta.rs +++ b/crates/model/tests/solar_open2_meta.rs @@ -350,6 +350,7 @@ fn hw_solar_open2_plan_a() { staging: 0, scratch: 0, host: 0, + leg: None, }; let bp = workstation::plan_bp(layers, None, tier_batch); let tiered = inputs diff --git a/crates/placement/src/accepts.rs b/crates/placement/src/accepts.rs index 1afdc9890..20c293dbc 100644 --- a/crates/placement/src/accepts.rs +++ b/crates/placement/src/accepts.rs @@ -416,6 +416,43 @@ pub fn leg_runs(p: Program, [gate, up, down]: [GgmlType; 3]) -> Option { leg_arms(p).map(|a| gate == up && (a.gate_up)(gate) && (a.down)(down)) } +/// The union of down activation forms of the complete routed layers that +/// `p`'s placement rule admits, resolved from the file's descriptor at load. +#[must_use] +pub fn leg_down_forms(p: Program, model: &crate::placement::ModelTensors) -> kernels::DownForms { + use crate::placement::{ExpertPart, Role}; + + let RoutedRule::Leg { gate_up, down } = routed_rule(p) else { + return kernels::DownForms::NONE; + }; + let mut forms = kernels::DownForms::NONE; + for l in 0..model.layers { + let mut parts = [None; 3]; + for t in model + .tensors + .iter() + .filter(|t| t.role == Role::RoutedExperts && t.layer == Some(l)) + { + let part = match ExpertPart::of_name(&t.name) { + Some(ExpertPart::Gate) => 0, + Some(ExpertPart::Up) => 1, + Some(ExpertPart::Down) => 2, + None => continue, + }; + parts[part] = Some(t.ty); + } + if let [Some(g), Some(u), Some(d)] = parts + && g == u + && gate_up(g).is_some() + && down(d).is_some() + && let Some((_, act)) = kernels::down(d) + { + forms = forms.with(act); + } + } + forms +} + /// Program `p`'s plan rule for its routed stacks ([`RoutedRule`]): on a card /// leg, each part by the arm its load reads ([`leg_runs`]), so a layer the /// plan gives card experts is one the load runs. The common leg's plan, over diff --git a/crates/placement/src/kernels.rs b/crates/placement/src/kernels.rs index 8b9d69940..7bd7941c4 100644 --- a/crates/placement/src/kernels.rs +++ b/crates/placement/src/kernels.rs @@ -100,12 +100,30 @@ pub const fn embed(ty: GgmlType) -> Option { /// it holds; a form whose last super-block is padded passes the padded width. #[must_use] pub const fn q8_act_bytes(cols: u64, k: u64) -> u64 { - let n_sb = k / 256; - 8 * cols * 64 * n_sb.div_ceil(2) - + 4 * cols * 256 * n_sb.div_ceil(4) - + 4 * cols * 128 * n_sb.div_ceil(2) - + 4 * cols * 8 * n_sb - + 4 * cols * 2 * n_sb + match try_q8_act_bytes(cols, k) { + Some(bytes) => bytes, + None => panic!("q8_act_bytes: activation bytes overflow u64"), + } +} + +/// [`q8_act_bytes`] with overflow returned to the placement boundary. +#[must_use] +pub(crate) const fn try_q8_act_bytes(cols: u64, k: u64) -> Option { + let n_sb = (k / 256) as u128; + // A u64 width's per-column count fits u128; its column product may not. + let per_col = 512 * n_sb.div_ceil(2) + + 1024 * n_sb.div_ceil(4) + + 512 * n_sb.div_ceil(2) + + 32 * n_sb + + 8 * n_sb; + let Some(bytes) = per_col.checked_mul(cols as u128) else { + return None; + }; + if bytes > u64::MAX as u128 { + None + } else { + Some(bytes as u64) + } } /// The card bytes of the 32-value q8_1 activation of `cols` columns of `k` @@ -114,8 +132,25 @@ pub const fn q8_act_bytes(cols: u64, k: u64) -> u64 { /// and scales, four bytes each, `k_blocks = k / 32`. #[must_use] pub const fn q8_blocks32_bytes(cols: u64, k: u64) -> u64 { - let k_blocks = k / 32; - cols * 4 * (256 * k_blocks.div_ceil(32) + 2 * k_blocks) + match try_q8_blocks32_bytes(cols, k) { + Some(bytes) => bytes, + None => panic!("q8_blocks32_bytes: activation bytes overflow u64"), + } +} + +/// [`q8_blocks32_bytes`] with overflow returned to the placement boundary. +#[must_use] +const fn try_q8_blocks32_bytes(cols: u64, k: u64) -> Option { + let k_blocks = (k / 32) as u128; + let per_col = 4 * (256 * k_blocks.div_ceil(32) + 2 * k_blocks); + let Some(bytes) = per_col.checked_mul(cols as u128) else { + return None; + }; + if bytes > u64::MAX as u128 { + None + } else { + Some(bytes as u64) + } } /// The down activation forms ([`DownAct`]) a card leg's buffers hold: the @@ -174,30 +209,68 @@ impl DownForms { /// them read it. #[must_use] pub const fn leg_bytes(cap: u64, slots_per_col: u64, n: u64, ff: u64, downs: DownForms) -> u64 { - let slots = cap * slots_per_col; + match try_leg_bytes(cap, slots_per_col, n, ff, downs) { + Some(bytes) => bytes, + None => panic!("leg_bytes: scratch bytes overflow u64"), + } +} + +/// [`leg_bytes`] with overflow returned to the placement boundary. This +/// counts the same whole blocks and padded planes as the infallible sizing. +#[must_use] +pub(crate) const fn try_leg_bytes( + cap: u64, + slots_per_col: u64, + n: u64, + ff: u64, + downs: DownForms, +) -> Option { + let Some(slots) = cap.checked_mul(slots_per_col) else { + return None; + }; + let Some(row_bytes) = slots.checked_mul(4) else { + return None; + }; + let Some(h) = row_bytes.checked_mul(ff) else { + return None; + }; + let Some(down) = row_bytes.checked_mul(n) else { + return None; + }; + let Some(input) = try_q8_act_bytes(cap, n) else { + return None; + }; let q8 = if downs.q8_act { - q8_act_bytes(slots, ff) + let Some(bytes) = try_q8_act_bytes(slots, ff) else { + return None; + }; + bytes } else { 0 }; let b32 = if downs.blocks32 { - q8_blocks32_bytes(slots, ff) + let Some(bytes) = try_q8_blocks32_bytes(slots, ff) else { + return None; + }; + bytes } else { 0 }; - q8_act_bytes(cap, n) + 4 * slots * ff + q8 + b32 + 4 * slots * n + let bytes = input as u128 + h as u128 + q8 as u128 + b32 as u128 + down as u128; + if bytes > u64::MAX as u128 { + None + } else { + Some(bytes as u64) + } } /// Whether the common card leg (`crates/gpu/src/card_leg.rs`) launches the /// gate·up entry of a routed gate or up stack of type `ty` in the file's /// blocks ([`crate::placement::CardFormat::KQuant`]): the table's [`gate_up`] /// row, for the entries whose launchers read a `Q8Act` of more columns than a -/// run uses. The Q8_0 row has a launcher of that shape and an arm in the leg -/// (`CardLeg::launch`), which this set leaves out: the families that plan by -/// it (GLM-5.3, MiMo, Solar-Open2, MiniMax-M3, Raon) hold `Q8Act` down buffers -/// only, so a Q8_0 layer they planned on the card would fail at its first -/// launch where it runs on the host today; a program whose buffers follow the -/// down's form reads the arm through its own row ([`crate::accepts::prompt_leg_gate_up`]). +/// run uses. The Q8_0 and Q6_K rows have launchers of that shape and arms in the +/// leg, but only programs whose selected gate·up policy admits them read those +/// arms ([`crate::accepts::prompt_leg_gate_up`]). /// A stack in its planes is [`card_planes_runs`]'. #[must_use] pub const fn card_gate_up_runs(ty: GgmlType) -> bool { @@ -323,6 +396,54 @@ mod tests { use super::*; use crate::placement::CardFormat; + /// Plane-padding transitions retain their byte counts, and both sizing + /// contracts share the last representable column and aggregate boundary. + #[test] + fn activation_and_leg_bytes_keep_padding_and_overflow_boundaries() { + for (k, bytes) in [(1024, 3232), (1280, 5320)] { + assert_eq!(try_q8_act_bytes(1, k), Some(bytes)); + assert_eq!(q8_act_bytes(1, k), bytes); + } + for (k, bytes) in [(1024, 1280), (1056, 2312)] { + assert_eq!(try_q8_blocks32_bytes(1, k), Some(bytes)); + assert_eq!(q8_blocks32_bytes(1, k), bytes); + } + let q8 = DownForms::of(DownAct::Q8Act); + let b32 = DownForms::of(DownAct::Q8Blocks32); + for (forms, bytes) in [ + (DownForms::NONE, 4136), + (q8, 6224), + (b32, 5224), + (q8.with(DownAct::Q8Blocks32), 7312), + ] { + assert_eq!(try_leg_bytes(1, 1, 256, 256, forms), Some(bytes)); + assert_eq!(leg_bytes(1, 1, 256, 256, forms), bytes); + } + assert_eq!(try_leg_bytes(1, 1, 64, 24, DownForms::NONE), Some(352)); + let act_cols = u64::MAX / 2088; + assert_eq!(try_q8_act_bytes(act_cols, 256), Some(act_cols * 2088)); + assert_eq!(q8_act_bytes(act_cols, 256), act_cols * 2088); + assert_eq!(try_q8_act_bytes(act_cols + 1, 256), None); + let block_cols = u64::MAX / 1088; + assert_eq!( + try_q8_blocks32_bytes(block_cols, 256), + Some(block_cols * 1088) + ); + assert_eq!(q8_blocks32_bytes(block_cols, 256), block_cols * 1088); + assert_eq!(try_q8_blocks32_bytes(block_cols + 1, 256), None); + let cap = u64::MAX / 6224; + assert_eq!(try_leg_bytes(cap, 1, 256, 256, q8), Some(cap * 6224)); + assert_eq!(leg_bytes(cap, 1, 256, 256, q8), cap * 6224); + assert_eq!(try_leg_bytes(cap + 1, 1, 256, 256, q8), None); + assert_eq!(try_leg_bytes(1 << 61, 8, 256, 256, q8), None); + } + + #[test] + #[should_panic(expected = "leg_bytes: scratch bytes overflow u64")] + fn infallible_leg_sizing_names_an_overflow() { + let _ = leg_bytes(1 << 61, 8, 256, 256, DownForms::of(DownAct::Q8Act)); + } + /// Every `GgmlType` variant in declaration order, then one id the enum /// does not name. const ALL: [GgmlType; 42] = [ diff --git a/crates/placement/src/placement.rs b/crates/placement/src/placement.rs index 5a739de5b..ee6acf9ac 100644 --- a/crates/placement/src/placement.rs +++ b/crates/placement/src/placement.rs @@ -227,6 +227,8 @@ pub struct Card { /// draft model's resident bytes, say): taken from the budget with the /// context and the scratch ([`Card::set_aside_bytes`]). pub reserves: Vec<(String, u64)>, + /// A tier leg's scratch, jointly selected with its resident experts. + pub tier_leg: Option, } impl Card { @@ -248,10 +250,55 @@ impl Card { .try_fold(0u64, |sum, (_, bytes)| sum.checked_add(*bytes)) } - /// What the card sets aside before any tensor or cache: context, - /// scratch and reserves. The one owner of that sum: the expert rule's - /// budget, the budget floor, the headroom and [`Plan::violations`] all - /// take it from here. The margin is not in it. `None` past u64 bytes. + /// The tier leg's bound and selected scratch; absent on fixed-reserve paths. + fn selected_scratch( + &self, + model: &ModelTensors, + counts: &[u64], + ) -> Result, PlacementError> { + let Some(leg) = self.tier_leg else { + return Ok(None); + }; + let forms = model + .tensors + .iter() + .filter(|t| { + t.role == Role::RoutedExperts + && t.layer + .is_some_and(|l| counts.get(l).is_some_and(|&n| n > 0)) + && ExpertPart::of_name(&t.name) == Some(ExpertPart::Down) + }) + .filter_map(|t| crate::kernels::down(t.ty)) + .fold(crate::kernels::DownForms::NONE, |forms, (_, act)| { + forms.with(act) + }); + Ok(Some((leg.bytes(leg.downs)?, leg.bytes(forms)?))) + } + + fn selected_reserve_bytes( + &self, + model: &ModelTensors, + counts: &[u64], + ) -> Result { + let fixed = self.reserve_bytes().ok_or_else(|| self.bytes_past_u64())?; + match self.selected_scratch(model, counts)? { + Some((bound, selected)) => fixed + .checked_sub(bound) + .and_then(|n| n.checked_add(selected)) + .ok_or_else(|| self.bytes_past_u64()), + None => Ok(fixed), + } + } + + fn floor_with_reserves(&self, dense: u64, kv: u64, past: u64, reserves: u64) -> Option { + [kv, self.context_bytes, self.scratch_bytes, reserves, past] + .into_iter() + .try_fold(dense, u64::checked_add) + } + + /// The machine's context, scratch and reserve bound before uploads. + /// Placement replaces a tier leg's bound with its candidate's actual + /// scratch. The margin is not in it. `None` past u64 bytes. #[must_use] pub fn set_aside_bytes(&self) -> Option { self.context_bytes @@ -259,18 +306,13 @@ impl Card { .checked_add(self.reserve_bytes()?) } - /// The card's floor under `dense` bytes of uploads and `kv` bytes of - /// cache: those, its set-aside, and `past` — what the load keeps beside - /// them: a plan's margin and what its caller holds beside the plan - /// (`plan_floor` sums those), or a whole load's own arena and reserve - /// ([`WholeLoad::past_bytes`]). The one owner of that sum: the budget - /// floor, the free floor and the whole fit ([`whole_need`]) all take it - /// from here. `None` past u64 bytes. + /// The fixed/bounded floor under `dense` upload bytes, `kv` cache bytes + /// and `past` bytes held beside them ([`WholeLoad::past_bytes`]). + /// Selected plans use the same sum with their actual reserves. + /// `None` past u64 bytes. #[must_use] pub fn floor_bytes(&self, dense: u64, kv: u64, past: u64) -> Option { - [kv, self.set_aside_bytes()?, past] - .into_iter() - .try_fold(dense, u64::checked_add) + self.floor_with_reserves(dense, kv, past, self.reserve_bytes()?) } fn bytes_past_u64(&self) -> PlacementError { @@ -897,7 +939,8 @@ pub struct CardTotals { pub stage_rows_bytes: u64, pub scratch_bytes: u64, pub context_bytes: u64, - /// The card's reserves ([`Card::reserves`]). + /// The selected reserves; a tier leg's descriptor bound is replaced by + /// its actual resident layers' scratch. pub reserve_bytes: u64, /// usable − dense − experts − rounding − KV − stage rows − scratch − /// context − reserves, usable capped by the plan's card budget; the @@ -2572,6 +2615,40 @@ impl Fill<'_, '_> { .routed .unwrap_or(RoutedRule::PerType(CardFormat::of_routed)); let uploads = card_uploads(self.model, self.c, self.rows, self.eligible, format)?; + let mut down_acts = if card.tier_leg.is_some() { + vec![None; self.model.layers] + } else { + Vec::new() + }; + if card.tier_leg.is_some() { + for t in &self.model.tensors { + if t.role == Role::RoutedExperts + && ExpertPart::of_name(&t.name) == Some(ExpertPart::Down) + && let Some(l) = t.layer + && let Some((_, act)) = crate::kernels::down(t.ty) + { + down_acts[l] = Some(act); + } + } + } + let scratch = |counts: &[u64]| { + card.tier_leg.map_or(Ok(0), |leg| { + let forms = counts + .iter() + .zip(&down_acts) + .filter_map(|(&n, &act)| (n > 0).then_some(act).flatten()) + .fold(crate::kernels::DownForms::NONE, |forms, act| { + forms.with(act) + }); + leg.bytes(forms) + }) + }; + let bound = card + .tier_leg + .map(|leg| leg.bytes(leg.downs)) + .transpose()? + .unwrap_or(0); + let reserves = card.selected_reserve_bytes(self.model, n_l)?; let budget = i128::from(capped(card, self.card_budget)) - i128::from(self.kv) - i128::from( @@ -2579,20 +2656,23 @@ impl Fill<'_, '_> { .ok_or_else(|| card.bytes_past_u64())?, ) - i128::from(card.margin_bytes) - - i128::from(self.reserve); + - i128::from(self.reserve) + + i128::from(bound); if let Some(b) = self.card_budget { let dense = footprint(card.granule_bytes, &uploads, n_l)?; - check_floor(card, b, dense, self.kv, self.reserve)?; + check_floor(card, b, dense, self.kv, self.reserve, reserves)?; } if let Some(free) = card.free_bytes.filter(|&f| f < card.usable_bytes) { let dense = footprint(card.granule_bytes, &uploads, n_l)?; - check_free_floor(card, free, dense, self.kv, self.reserve)?; + check_free_floor(card, free, dense, self.kv, self.reserve, reserves)?; } let caps: Vec = (0..n_l.len()) .map(|l| self.model.layer_experts(l)) .collect(); spread(n_l, held, self.eligible, &caps, budget, |n| { - footprint(card.granule_bytes, &uploads, n) + footprint(card.granule_bytes, &uploads, n)? + .checked_add(scratch(n)?) + .ok_or_else(|| card.bytes_past_u64()) }) } } @@ -2604,10 +2684,10 @@ impl Fill<'_, '_> { /// least budget at which the expert rule's own budget (usable less KV, /// set-aside, margin and `beside`, in [`Fill::run`]) reaches the dense /// uploads. `None` past u64 bytes. -fn plan_floor(card: &Card, dense: u64, kv: u64, beside: u64) -> Option { +fn plan_floor(card: &Card, dense: u64, kv: u64, beside: u64, reserves: u64) -> Option { card.margin_bytes .checked_add(beside) - .and_then(|past| card.floor_bytes(dense, kv, past)) + .and_then(|past| card.floor_with_reserves(dense, kv, past, reserves)) } /// Refuse `card` under the card budget `budget` when its floor — the @@ -2622,8 +2702,9 @@ fn check_floor( dense: u64, kv: u64, beside: u64, + reserves: u64, ) -> Result<(), PlacementError> { - let floor = plan_floor(card, dense, kv, beside); + let floor = plan_floor(card, dense, kv, beside, reserves); match floor { Some(floor) if floor <= budget => Ok(()), _ => Err(PlacementError::CardBudgetFloor { @@ -2634,7 +2715,7 @@ fn check_floor( kv, context: card.context_bytes, scratch: card.scratch_bytes, - reserves: card.reserve_bytes().ok_or_else(|| card.bytes_past_u64())?, + reserves, margin: card.margin_bytes, beside, }), @@ -2653,9 +2734,10 @@ fn check_free_floor( dense: u64, kv: u64, beside: u64, + reserves: u64, ) -> Result<(), PlacementError> { let context = card.context_bytes - card.context_in_free(); - let floor = plan_floor(card, dense, kv, beside).map(|f| f - card.context_in_free()); + let floor = plan_floor(card, dense, kv, beside, reserves).map(|f| f - card.context_in_free()); match floor { Some(floor) if floor <= free => Ok(()), floor => Err(PlacementError::CardFreeFloor(Box::new(CardFreeFloor { @@ -2668,7 +2750,7 @@ fn check_free_floor( kv, context, scratch: card.scratch_bytes, - reserves: card.reserve_bytes().ok_or_else(|| card.bytes_past_u64())?, + reserves, margin: card.margin_bytes, beside, held: card @@ -3751,8 +3833,12 @@ fn totals<'a>( .map(|(c, (card, heap))| { let rounding = heap.taken - (card_dense[c] + card_experts[c]); let (kv, _, stage_rows) = kv_bytes[c]; + let counts = c + .checked_sub(machine.cards.len()) + .map_or(n_l.as_slice(), |t| tier_n_l[t].as_slice()); + let reserve_bytes = card.selected_reserve_bytes(model, counts)?; let used = card - .floor_bytes(heap.taken, kv, stage_rows) + .floor_with_reserves(heap.taken, kv, stage_rows, reserve_bytes) .ok_or_else(|| card.bytes_past_u64())?; if machine.unified().is_some() && machine.on_this_node(c) @@ -3772,9 +3858,7 @@ fn totals<'a>( stage_rows_bytes: stage_rows, scratch_bytes: card.scratch_bytes, context_bytes: card.context_bytes, - reserve_bytes: card - .reserve_bytes() - .expect("the card floor checked its reserves"), + reserve_bytes, headroom_bytes: i128::from(capped(card, card_budget)) - i128::from(used), }) }) @@ -3843,11 +3927,50 @@ pub struct PoolNeed { /// `card`'s total against usable − margin: its uploads' granules, its KV, /// its stage rows and its set-aside. fn card_total(card: &Card, totals: &CardTotals, heap: &Heap) -> u64 { - card.floor_bytes(heap.taken, totals.kv_bytes, totals.stage_rows_bytes) - .expect("the plan's card totals fit u64") + card.floor_with_reserves( + heap.taken, + totals.kv_bytes, + totals.stage_rows_bytes, + totals.reserve_bytes, + ) + .expect("the plan's card totals fit u64") } impl Plan<'_> { + /// One named reserve at its selected amount, not the machine's scratch + /// bound. Missing or duplicate rows and an invalid card return `None`; + /// invalid scratch descriptors propagate their named error. + pub fn card_reserve(&self, c: usize, name: &str) -> Result, PlacementError> { + let Some(card) = self.machine.card(c) else { + return Ok(None); + }; + let mut rows = card.reserves.iter().filter(|(n, _)| n == name); + let Some((_, bytes)) = rows.next() else { + return Ok(None); + }; + if rows.next().is_some() { + return Ok(None); + } + if name != workstation::TIER_BATCH_RESERVE { + return Ok(Some(*bytes)); + } + let counts = match c.checked_sub(self.machine.cards.len()) { + None => self.n_l.as_slice(), + Some(t) => match self.tier_n_l.get(t) { + Some(counts) => counts.as_slice(), + None => return Ok(None), + }, + }; + match card.selected_scratch(self.model, counts)? { + Some((bound, selected)) => bytes + .checked_sub(bound) + .and_then(|bytes| bytes.checked_add(selected)) + .map(Some) + .ok_or_else(|| card.bytes_past_u64()), + None => Ok(Some(*bytes)), + } + } + fn reserve_beside(&mut self, reserve: u64) -> Result<(), PlacementError> { if self.machine.unified().is_some() { let here = (0..self.machine.all_cards().count()) @@ -4469,6 +4592,7 @@ mod tests { fn card(name: &str, layers: Range, head: bool, token_embedding: bool) -> Card { Card { name: name.to_string(), + tier_leg: None, device: None, usable_bytes: 0, context_bytes: 0, @@ -5574,6 +5698,169 @@ mod tests { assert_eq!(plan.n_l, vec![8]); } + /// Scratch enters each expert candidate's footprint monotonically. + /// A boundary that admits one down form cannot pay for an unused second + /// form, and adding the first expert of that form pays its exact bytes. + #[test] + fn a_tier_jointly_selects_expert_rows_and_down_forms() { + use crate::accepts::{Program, routed_rule}; + use crate::kernels::{DownAct, DownForms}; + use GgmlType::{Q5_K, Q8_0}; + use workstation::{TIER_BATCH_RESERVE, tier_leg_batch_bytes}; + + let both = DownForms::of(DownAct::Q8Act).with(DownAct::Q8Blocks32); + let batch = tier_leg_batch_bytes(256, 256, 1, 2, 3, both).expect("tier bytes"); + let leg = batch.leg.expect("a tier leg descriptor"); + let rule = routed_rule(Program::Qwen35Body); + for (downs, first) in [ + ([Q5_K, Q8_0], DownAct::Q8Act), + ([Q8_0, Q5_K], DownAct::Q8Blocks32), + ] { + let mut model = legged(&[[Q5_K, Q5_K, downs[0]], [Q5_K, Q5_K, downs[1]]]); + model.experts = 1; + model.experts_used = 1; + for t in model + .tensors + .iter_mut() + .filter(|t| t.role == Role::RoutedExperts) + { + t.dims[1] = 256; + t.dims[2] = 1; + t.file_bytes *= 8; + } + let expert = |l| { + model + .tensors + .iter() + .filter(|t| t.role == Role::RoutedExperts && t.layer == Some(l)) + .map(|t| t.file_bytes / model.experts) + .sum::() + }; + let one = + batch.staging + leg.bytes(DownForms::of(first)).expect("first scratch") + expert(0); + let two = + batch.staging + leg.bytes(both).expect("mixed scratch") + expert(0) + expert(1); + for (room, counts, forms) in [ + (one - 1, [0, 0], DownForms::NONE), + (one, [1, 0], DownForms::of(first)), + (two - 1, [1, 0], DownForms::of(first)), + (two, [1, 1], both), + ] { + let mut tier = bytes_card("tier", room, 0..0); + tier.reserves + .push((TIER_BATCH_RESERVE.into(), batch.card().unwrap())); + tier.tier_leg = batch.leg; + let machine = Machine { + cards: vec![bytes_card("stage", HEAD, 0..2)], + tiers: vec![tier], + nodes: vec![Node { + host: host(), + unified: None, + }], + }; + let plan = plan_routed(&model, &machine, 1, &NoKv, &PlanLevers::default(), rule) + .expect("the joint plan"); + assert_eq!(plan.n_l, [0, 0]); + assert_eq!(plan.tier_n_l[0], counts, "{downs:?}, {room}"); + let selected = batch.staging + leg.bytes(forms).expect("selected scratch"); + assert_eq!( + plan.card_reserve(1, TIER_BATCH_RESERVE) + .expect("selected reserve"), + Some(selected) + ); + assert_eq!(plan.cards[1].reserve_bytes, selected); + let rows = expert(0) * counts[0] + expert(1) * counts[1]; + assert_eq!(plan.cards[1].bytes(), selected + rows); + assert!(plan.violations().is_empty(), "{downs:?}, {room}"); + } + } + } + + /// A malformed descriptor must refuse a plan and a later selected-reserve + /// read, not wrap to a small scratch or masquerade as a fixed-reserve card. + #[test] + fn a_tier_refuses_invalid_scratch_descriptors_before_placement() { + use crate::accepts::{Program, routed_rule}; + use crate::kernels::{DownAct, DownForms}; + use workstation::{TIER_BATCH_RESERVE, TierLegScratch, tier_leg_batch_bytes}; + + let forms = DownForms::of(DownAct::Q8Act); + let batch = tier_leg_batch_bytes(256, 256, 1, 1, 1, forms).expect("valid batch"); + let good = batch.leg.expect("a leg descriptor"); + let mut model = legged(&[[GgmlType::Q5_K; 3]]); + model.experts_used = 1; + for t in model + .tensors + .iter_mut() + .filter(|t| t.role == Role::RoutedExperts) + { + t.dims[1] = 256; + t.file_bytes *= 64; + } + let machine = |leg| { + let mut tier = bytes_card("tier", 1 << 30, 0..0); + tier.reserves + .push((TIER_BATCH_RESERVE.into(), batch.card().unwrap())); + tier.tier_leg = Some(leg); + Machine { + cards: vec![bytes_card("stage", HEAD, 0..1)], + tiers: vec![tier], + nodes: vec![Node { + host: host(), + unified: None, + }], + } + }; + let valid = machine(good); + let rule = routed_rule(Program::Qwen35Body); + let plan = plan_routed(&model, &valid, 1, &NoKv, &PlanLevers::default(), rule) + .expect("valid descriptor plans"); + for leg in [ + TierLegScratch { + chunk: 1 << 61, + n_used: 8, + ..good + }, + TierLegScratch { chunk: 0, ..good }, + TierLegScratch { ff: 0, ..good }, + TierLegScratch { n_embd: 0, ..good }, + ] { + let bad = machine(leg); + assert!( + matches!( + plan_routed(&model, &bad, 1, &NoKv, &PlanLevers::default(), rule), + Err(PlacementError::Metadata { key, .. }) if key == "tier leg scratch" + ), + "{leg:?}" + ); + let mut stale = plan.clone(); + stale.machine = &bad; + assert!( + matches!( + stale.card_reserve(1, TIER_BATCH_RESERVE), + Err(PlacementError::Metadata { key, .. }) if key == "tier leg scratch" + ), + "{leg:?}" + ); + assert!( + matches!( + tier_leg_batch_bytes(leg.n_embd, leg.ff, leg.n_used, leg.chunk, 1, leg.downs), + Err(PlacementError::Metadata { key, .. }) if key == "tier leg scratch" + ), + "{leg:?}" + ); + } + assert!(tier_leg_batch_bytes(256, 256, 8, 1, 1 << 61, forms).is_err()); + let inconsistent = workstation::TierBatchBytes { + scratch: 0, + ..batch + }; + assert!(matches!( + inconsistent.card(), + Err(PlacementError::Metadata { key, .. }) if key == "tier leg batch" + )); + } + // PIN(2026-10-22): the Qwen3.6 leg runs a Q8_0 down and (once the leg has the // arm) a Q6_K gate and up through the common leg, so the layers that held those // types hold types it still has no GEMM for (an MXFP4 down, an IQ2_XS gate and diff --git a/crates/placement/src/placement/nodes_scale.rs b/crates/placement/src/placement/nodes_scale.rs index 67dfcbcb7..20d9c1bbf 100644 --- a/crates/placement/src/placement/nodes_scale.rs +++ b/crates/placement/src/placement/nodes_scale.rs @@ -283,6 +283,7 @@ fn census_budgets_keep_each_node_and_stage_terms() { staging: 123, scratch: 456, host: 789, + leg: None, }, ) .expect("three independent nodes"); @@ -339,6 +340,7 @@ fn cases() -> &'static [Case] { staging: 0, scratch: 0, host: 0, + leg: None, }, ) .expect("two nodes with no batch bytes"); diff --git a/crates/placement/src/placement/qwen38_cards.rs b/crates/placement/src/placement/qwen38_cards.rs index cd219e349..47bb5b9cb 100644 --- a/crates/placement/src/placement/qwen38_cards.rs +++ b/crates/placement/src/placement/qwen38_cards.rs @@ -249,6 +249,7 @@ const CHECKPOINT_38: u64 = 36 * 3_596_288 + 17 * 4 * 2560 * 4; fn stage(spec: CardSpec) -> Card { Card { name: spec.name.to_string(), + tier_leg: None, device: spec.device, usable_bytes: spec.usable_bytes(), context_bytes: CONTEXT, @@ -293,6 +294,7 @@ fn machine_bp(stage_spec: CardSpec, tier: CardSpec) -> Machine { cards: vec![s], tiers: vec![Card { name: tier.name.to_string(), + tier_leg: None, device: tier.device, usable_bytes: tier.usable_bytes(), context_bytes: CONTEXT, diff --git a/crates/placement/src/placement/workstation.rs b/crates/placement/src/placement/workstation.rs index ffcae22b4..40419caa4 100644 --- a/crates/placement/src/placement/workstation.rs +++ b/crates/placement/src/placement/workstation.rs @@ -17,7 +17,7 @@ pub use super::devices::{ #[cfg(test)] use super::nodes::{Fabric, Nodes}; use super::{Card, Host, HostTotals, Machine, Node, PlacementError, Plan, UnifiedPool}; -use crate::kernels::{DownAct, DownForms, leg_bytes, q8_act_bytes}; +use crate::kernels::{DownForms, leg_bytes, q8_act_bytes, try_leg_bytes, try_q8_act_bytes}; pub const MIB: u64 = 1 << 20; @@ -164,6 +164,7 @@ pub fn model_v41() -> String { fn card(spec: CardSpec, layers: Range, head: bool) -> Card { Card { name: spec.name.to_string(), + tier_leg: None, device: spec.device, usable_bytes: spec.usable_bytes(), context_bytes: CONTEXT, @@ -192,6 +193,7 @@ pub fn tier_card(spec: CardSpec, draft_bytes: Option, batch: TierBatchBytes .card() .unwrap_or_else(|error| panic!("tier batch card reserve: {error}")), )); + tier.tier_leg = batch.leg; tier } @@ -307,6 +309,7 @@ pub fn plan_a_draft(layers: usize, draft_bytes: Option) -> Result, } impl TierBatchBytes { - /// The tier card's part; refuses a sum past u64. + /// The tier card's part; refuses an invalid leg bound or a sum past u64. pub fn card(&self) -> Result { + if let Some(leg) = self.leg + && leg.bytes(leg.downs)? != self.scratch + { + return Err(PlacementError::Metadata { + key: "tier leg batch".into(), + detail: "scratch reserve differs from the descriptor's bound".into(), + }); + } self.staging .checked_add(self.scratch) .ok_or(PlacementError::TierBatchCard { @@ -371,6 +384,42 @@ impl TierBatchBytes { } } +/// The common leg's scratch geometry and the descriptor's admitted forms. +/// Placement prices its candidate's selected forms jointly with expert rows. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TierLegScratch { + pub n_embd: u64, + pub ff: u64, + pub n_used: u64, + pub chunk: u64, + pub downs: DownForms, +} + +impl TierLegScratch { + /// Scratch for exactly these selected forms, refusing invalid geometry + /// and arithmetic overflow before an allocation or placement candidate. + pub fn bytes(self, downs: DownForms) -> Result { + let invalid = || PlacementError::Metadata { + key: "tier leg scratch".into(), + detail: format!("invalid or overflowing geometry: {self:?}, selected {downs:?}"), + }; + if self.chunk == 0 + || self.n_used == 0 + || self.ff == 0 + || self.n_embd == 0 + || [ + crate::kernels::DownAct::Q8Act, + crate::kernels::DownAct::Q8Blocks32, + ] + .into_iter() + .any(|act| downs.holds(act) && !self.downs.holds(act)) + { + return Err(invalid()); + } + try_leg_bytes(self.chunk, self.n_used, self.n_embd, self.ff, downs).ok_or_else(invalid) + } +} + /// Experts the tile path's bucket table takes a layer at most, and the /// columns a tile holds: the tile path's shapes, which size its scratch. const TILE_BUCKET_EXPERTS: u64 = 1024; @@ -430,12 +479,18 @@ pub const fn walk_a_tile_scratch_bytes(n_embd: u64, ff: u64, n_used: u64, cols: /// The tier's chunk scratch for blocks of rows of `n_embd`, `n_used` slots a /// token, through experts of `ff`, cut in chunks of `chunk` tokens: one run of -/// the card leg's buffers (`bloomery_gpu::card_leg::LegBufs`, the GLM-5.3-Flash -/// tier) with its slots' columns in the 128-value q8_1 form the tier's downs -/// read ([`leg_bytes`]), in place of the tile path's scratch. +/// the card leg's buffers (`bloomery_gpu::card_leg::LegBufs`) with its slots' +/// columns in exactly the forms the selected downs read ([`leg_bytes`]), +/// in place of the tile path's scratch. #[must_use] -pub const fn tier_chunk_scratch_bytes(n_embd: u64, ff: u64, n_used: u64, chunk: u64) -> u64 { - leg_bytes(chunk, n_used, n_embd, ff, DownForms::of(DownAct::Q8Act)) +pub const fn tier_chunk_scratch_bytes( + n_embd: u64, + ff: u64, + n_used: u64, + chunk: u64, + downs: DownForms, +) -> u64 { + leg_bytes(chunk, n_used, n_embd, ff, downs) } /// The host's part of the tier's prompt-batch service: per exchange set, the @@ -454,6 +509,7 @@ pub const fn tier_batch_bytes(n_embd: u64, ff: u64, n_used: u64, cols: u64) -> T staging: tier_batch_staging_bytes(n_embd, n_used, cols), scratch: tier_block_scratch_bytes(n_embd, ff, cols * n_used), host: tier_batch_host_bytes(n_embd, n_used, cols), + leg: None, } } @@ -461,19 +517,48 @@ pub const fn tier_batch_bytes(n_embd: u64, ff: u64, n_used: u64, cols: u64) -> T /// scratch is one run of the leg's buffers for `chunk` tokens ([`tier_chunk_scratch_bytes`]), in /// place of the tile path's. The one formula of those bytes for every family whose tier is the card /// leg; a family names its own `chunk` and widths. -#[must_use] -pub const fn tier_leg_batch_bytes( +pub fn tier_leg_batch_bytes( n_embd: u64, ff: u64, n_used: u64, chunk: u64, cols: u64, -) -> TierBatchBytes { - TierBatchBytes { - staging: tier_batch_staging_bytes(n_embd, n_used, cols), - scratch: tier_chunk_scratch_bytes(n_embd, ff, n_used, chunk), - host: tier_batch_host_bytes(n_embd, n_used, cols), - } + downs: DownForms, +) -> Result { + let leg = TierLegScratch { + n_embd, + ff, + n_used, + chunk, + downs, + }; + let scratch = leg.bytes(downs)?; + let counted = || -> Option<(u64, u64)> { + if cols == 0 { + return None; + } + let slots = cols.checked_mul(n_used)?; + let places = 4u64.checked_mul(slots)?; + let down = places.checked_mul(n_embd)?; + let staging = 4u64 + .checked_mul(cols)? + .checked_mul(n_embd)? + .checked_add(places)? + .checked_add(try_q8_act_bytes(cols, n_embd)?)? + .checked_add(down)?; + let host = 2u64.checked_mul(down.checked_add(places)?)?; + Some((staging, host)) + }; + let (staging, host) = counted().ok_or_else(|| PlacementError::Metadata { + key: "tier leg batch".into(), + detail: format!("invalid or overflowing staging: {leg:?}, columns {cols}"), + })?; + Ok(TierBatchBytes { + staging, + scratch, + host, + leg: Some(leg), + }) } /// Plan (b′): the A6000 runs all `layers` and the head as in [`plan_a`]; @@ -551,6 +636,7 @@ pub fn plan_node_tiers( remote.cards[0] .reserves .extend(tier_card(spec, None, batch).reserves); + remote.cards[0].tier_leg = batch.leg; machines.push(remote); } Ok(machines) @@ -701,6 +787,7 @@ pub(crate) fn spark_nodes(n: usize, layers: usize, fabric: Fabric) -> Nodes { staging: 0, scratch: 0, host: 0, + leg: None, }; let mut machines = plan_node_tiers(layers, spec, &vec![spec; n - 1], batch) .expect("synthetic nodes with no batch bytes"); @@ -1578,6 +1665,7 @@ mod tests { staging: 1, scratch: 2, host: 3, + leg: None, }, ]; for layers in [0, 1, 20, 43, 47, 48, 61, 78] { @@ -1669,6 +1757,7 @@ mod tests { staging: u64::MAX, scratch: 1, host: 0, + leg: None, }; for result in [ plan_tiers(4, A6000, &[RTX_3090], None, huge).map(|_| ()), @@ -1719,6 +1808,7 @@ mod tests { staging: 10, scratch: 5, host: 100, + leg: None, }; let machines = plan_node_tiers(4, specs[0], &specs[1..], batch) .expect("four nodes of the same card name and ordinal"); @@ -1763,6 +1853,7 @@ mod tests { staging: 0, scratch: 0, host: u64::MAX / 2, + leg: None, }; let remotes = [gb10(80 << 30), gb10(60 << 30)]; let machines = @@ -1798,6 +1889,7 @@ mod tests { staging: 10, scratch: 5, host: 100, + leg: None, }; let m = plan_tiers(4, A6000, &[t1, t2], None, batch).expect("two tiers"); let rows: Vec<&(String, u64)> = m @@ -1878,6 +1970,7 @@ mod tests { assert_eq!(counted_arena_bytes(&m.cards[0]), 7, "{}", spec.name); let want = Card { name: spec.name.to_string(), + tier_leg: None, device: spec.device, usable_bytes: spec.usable_bytes(), context_bytes: CONTEXT, @@ -1925,6 +2018,7 @@ mod tests { staging: u64::MAX, scratch: 1, host: 0, + leg: None, }, ); } @@ -1944,6 +2038,7 @@ mod tests { staging: u64::MAX - 1, scratch: 1, host: 0, + leg: None, }; assert_eq!(edge.card().expect("the sum fits u64"), u64::MAX); assert!(matches!( diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 0d60c5feb..15def8b18 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -66,6 +66,25 @@ Each device slice needs its numerical/fault gates and the maintainer's offered h Legacy metadata booleans share `RouterNorm::from_legacy_bool`: false keeps scores and true uses the existing epsilon-added selected sum. Flint's distinct floor mapping stays in its reader. +### Selected expert arms and tier scratch + +Common card and tier wrappers resolve `LegArms` from their selected `Program`, rather than assuming +every family has one gate/up type or one down activation form. Their resident layers determine the +union of `Q8Act` and `Q8Blocks32` buffers. Plane layouts remain restricted to the selected program's +arms; Solar's host-only triples do not enlarge its card scratch. + +Placement charges a tier's selected down forms jointly with its expert count. The same kernels-owned, +checked sizing arithmetic is used by allocation and planning; malformed or overflowing public scratch +descriptors are refused by name. The plan records the selected reserve, and loaders check that amount +instead of the machine descriptor's upper bound. No always-both allocation or reserve-equality waiver +is used. + +A host-only public planner consumer exercised the one-byte boundaries before the first resident layer +and before adding the other down form, plus descriptor overflow refusal. This proves planning behavior, +not device allocation or expert arithmetic. The existing CUDA leg, batch and tier gates still need an +exact-head NVIDIA run. Flint's `Program` admission and body/app/seat remain absent. + + ## 1. Sources and observed artifact From a6beeace9c35aaad494ccb5669045c0eebf6de5c Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:12:33 +0900 Subject: [PATCH 02/19] placement: test overflow refusal without pinning panic wording --- crates/placement/src/kernels.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/placement/src/kernels.rs b/crates/placement/src/kernels.rs index 7bd7941c4..a7140ba70 100644 --- a/crates/placement/src/kernels.rs +++ b/crates/placement/src/kernels.rs @@ -439,8 +439,8 @@ mod tests { } #[test] - #[should_panic(expected = "leg_bytes: scratch bytes overflow u64")] - fn infallible_leg_sizing_names_an_overflow() { + #[should_panic] + fn infallible_leg_sizing_refuses_overflow() { let _ = leg_bytes(1 << 61, 8, 256, 256, DownForms::of(DownAct::Q8Act)); } From 2824f4fea029fa7c08f9c03b7747ac013e5ac86f Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:24:24 +0900 Subject: [PATCH 03/19] gpu: preserve fixed reserves for host-only node projections --- crates/gpu/src/host/node.rs | 21 +++++++++++++++------ docs/research/flint-attach.md | 3 +++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/crates/gpu/src/host/node.rs b/crates/gpu/src/host/node.rs index b15500b2f..a29006c95 100644 --- a/crates/gpu/src/host/node.rs +++ b/crates/gpu/src/host/node.rs @@ -320,14 +320,23 @@ impl NodeOpen { .find(|d| self.device.is_some_and(|id| id.uuid == d.uuid)) .ok_or_else(|| GpuError::shape(WHAT, "the projected tier is absent from the census"))?; let machine = workstation::plan_on(spec_of_device(device), self.layers); - let totals = plan.cards.get(self.card).ok_or_else(|| { + let card = plan.machine.card(self.card).ok_or_else(|| { GpuError::shape(WHAT, "the projected tier card is absent from the plan") })?; - let reserve = totals - .context_bytes - .checked_add(totals.scratch_bytes) - .and_then(|bytes| bytes.checked_add(totals.reserve_bytes)) - .ok_or_else(|| GpuError::shape(WHAT, "the projected tier reserve overflows"))?; + // Host-only projections have no card totals; their fixed reserves + // still belong to the machine. Dynamic scratch needs selected totals. + let reserve = if card.tier_leg.is_some() { + let totals = plan.cards.get(self.card).ok_or_else(|| { + GpuError::shape(WHAT, "the projected tier card is absent from the plan") + })?; + totals + .context_bytes + .checked_add(totals.scratch_bytes) + .and_then(|bytes| bytes.checked_add(totals.reserve_bytes)) + } else { + card.set_aside_bytes() + } + .ok_or_else(|| GpuError::shape(WHAT, "the projected tier reserve overflows"))?; let mut projected = model::placement::nodes::routed_host( plan.model, &plan.rows, diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 15def8b18..40ac1d498 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -78,6 +78,9 @@ checked sizing arithmetic is used by allocation and planning; malformed or overf descriptors are refused by name. The plan records the selected reserve, and loaders check that amount instead of the machine descriptor's upper bound. No always-both allocation or reserve-equality waiver is used. +Host-only projections intentionally have no card totals: fixed reserves remain owned by their machine. +Only a dynamic tier-leg descriptor requires the plan's selected card totals. The existing node paging +budget regression exercises this distinction without a GPU, including OS reserve and constrained host room. A host-only public planner consumer exercised the one-byte boundaries before the first resident layer and before adding the other down form, plus descriptor overflow refusal. This proves planning behavior, From b3a238017ba9c7578e8a8d41dbe0dd582aee880c Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:37:10 +0900 Subject: [PATCH 04/19] gpu: remove redundant wrapper test allocation and borrow --- crates/gpu-gates/src/bin/gate_solar2_twocard.rs | 2 +- crates/gpu/src/card_experts.rs | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/gpu-gates/src/bin/gate_solar2_twocard.rs b/crates/gpu-gates/src/bin/gate_solar2_twocard.rs index 9d413942a..24c1daab0 100644 --- a/crates/gpu-gates/src/bin/gate_solar2_twocard.rs +++ b/crates/gpu-gates/src/bin/gate_solar2_twocard.rs @@ -313,7 +313,7 @@ mod gate { let layers = inputs.model.layers; let bp = Place::Bp .on_host()? - .machine(None, Some(place::tier_batch(&inputs)?))?; + .machine(None, Some(place::tier_batch(inputs)?))?; let two = bp(layers); let plan = inputs.plan(&two, u64::try_from(CTX)?, &cfg.place)?; let budget = cfg diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index 4542fe4ac..8ed9d38f1 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -760,8 +760,7 @@ mod tests { let x = DeviceBuffer::from_host(stream, &host_x).expect("the rows"); let base: Vec = (0..CAP * N).map(|i| (i % 13) as f32 * 0.125).collect(); let x1 = DeviceBuffer::from_host(stream, &base).expect("the base"); - let route = - DeviceBuffer::from_host(stream, &vec![0.125f32; CAP * USED]).expect("weights"); + let route = DeviceBuffer::from_host(stream, &[0.125f32; CAP * USED]).expect("weights"); let host_sel: Vec = (0..CAP * USED) .map(|i| { if i % USED == USED - 1 { From a0e1e901a258e1c43fc7bdc4567c05f0e328751d Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 05:19:09 +0900 Subject: [PATCH 05/19] Reconcile selected scratch test contracts and format migrated callers --- .../src/bin/shared/serve_seats/minimaxm3.rs | 3 ++- crates/gpu/src/arch/qwen3moe/tier_leg.rs | 3 --- crates/gpu/src/card_experts.rs | 8 ++++++- crates/model/src/arch/glm5next/place.rs | 4 ++-- crates/model/src/arch/minimaxm3/place.rs | 24 +++---------------- crates/placement/src/accepts.rs | 7 +++++- crates/placement/src/placement.rs | 3 ++- crates/placement/src/placement/workstation.rs | 7 +----- 8 files changed, 23 insertions(+), 36 deletions(-) diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs b/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs index 27cd31f38..096093e07 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs @@ -505,7 +505,8 @@ impl Tiered for MiniMax { place: Place, inputs: &PlanInputs, ) -> Result, GateError> { - Ok((!place.tier_cards().is_empty()).then(|| bloomery_gpu_minimaxm3::tier::tier_batch(inputs))) + Ok((!place.tier_cards().is_empty()) + .then(|| bloomery_gpu_minimaxm3::tier::tier_batch(inputs))) } /// The tags the model's description names for its think span. diff --git a/crates/gpu/src/arch/qwen3moe/tier_leg.rs b/crates/gpu/src/arch/qwen3moe/tier_leg.rs index 5539b6dff..de8854b21 100644 --- a/crates/gpu/src/arch/qwen3moe/tier_leg.rs +++ b/crates/gpu/src/arch/qwen3moe/tier_leg.rs @@ -70,7 +70,6 @@ fn qwen_act(moe: &Moe) -> Result { } } - /// The host views of the routed layers `run`, the refusal naming the run as /// `word`'s. `n_layer` is the description's layer count, so a trunk layer is /// not refused for sitting past a next-token block. @@ -155,7 +154,6 @@ impl LegFamily for Qwen3 { fn tier_act(moe: &Moe) -> Result { qwen_act(moe) } - } /// Qwen3.6's facts for the common tier node service. The description's @@ -214,7 +212,6 @@ impl LegFamily for Qwen35 { fn tier_act(moe: &Moe) -> Result { qwen_act(moe) } - } /// Qwen3-30B's tier node service ([`leg_tier::node_family`]). diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index 23266831f..33e8afca2 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -1214,7 +1214,13 @@ mod tests { assert!( matches!( LegTier::new( - &gpu, &specs, &set, &w, family.shape(N, FF, 2), program, tier_act, + &gpu, + &specs, + &set, + &w, + family.shape(N, FF, 2), + program, + tier_act, ), Err(GpuError::Tensor { .. }), ), diff --git a/crates/model/src/arch/glm5next/place.rs b/crates/model/src/arch/glm5next/place.rs index 82efa5b6a..bca8663f6 100644 --- a/crates/model/src/arch/glm5next/place.rs +++ b/crates/model/src/arch/glm5next/place.rs @@ -1880,7 +1880,7 @@ mod tests { use GgmlType::{IQ3_S, IQ4_XS, NVFP4, Q4_K, Q5_K, Q6_K, Q8_0}; let q8 = DownForms::of(DownAct::Q8Act); let b32 = DownForms::of(DownAct::Q8Blocks32); - let base = tier_batch(&inputs()); + let base = tier_batch(&inputs()).expect("valid tier batch"); let hp = inputs().hp; let leg = |forms| { leg_bytes( @@ -1908,7 +1908,7 @@ mod tests { q8.with(DownAct::Q8Blocks32), ), ] { - let batch = tier_batch(&routed_types(&layers)); + let batch = tier_batch(&routed_types(&layers)).expect("valid selected tier batch"); assert_eq!(batch.scratch, leg(forms), "{layers:?}"); assert_eq!(batch.staging, base.staging, "{layers:?}"); assert_eq!(batch.host, base.host, "{layers:?}"); diff --git a/crates/model/src/arch/minimaxm3/place.rs b/crates/model/src/arch/minimaxm3/place.rs index 6cf405c84..309200f48 100644 --- a/crates/model/src/arch/minimaxm3/place.rs +++ b/crates/model/src/arch/minimaxm3/place.rs @@ -418,31 +418,13 @@ mod tests { )); } - /// A machine that names an expert tier card (plan (b′)) is planned: the body serves one. The - /// card keeps the prompt batch's reserve on either machine and, on the tiered one, the tier - /// places' copies beside it ([`tier_places_bytes`]: 3 × 4 B × 256 rows × 2 slots). The tier's batch bytes are the common leg's: runs of - /// [`TIER_CHUNK`] tokens over blocks of the unit's `cols` rows, in the forms the leg's routed - /// downs read: the header's F32 downs run on no leg, so the 128-value form - /// ([`workstation::tier_leg_batch_bytes`]). The plan reaches the placement, which refuses the - /// tier idle: the header's experts all fit the stage card. Mutant: the tier reserve left out - /// of the description (the plan stops at [`PlaceError::Tier`]), the forms of a file with no - /// leg down left empty (the scratch holds no slot columns), or the run's tokens and the - /// block's rows swapped. + /// A tiered machine retains the stage's prompt reserve and the tier places beside it. + /// The fixture's experts fit on the stage, so placement refuses the idle tier rather + /// than rejecting the description as a body without tier support. #[test] fn a_tier_card_is_planned_with_the_leg_batch() { const COLS: u64 = 3 * TIER_CHUNK; let inputs = described("mm3-place-tier"); - let hp = &inputs.hp; - assert_eq!( - tier_batch(&inputs, COLS), - workstation::tier_leg_batch_bytes( - hp.n_embd as u64, - hp.experts.ff as u64, - hp.experts.n_used as u64, - TIER_CHUNK, - COLS, - ) - ); let tiered = workstation::plan_bp(LAYERS, None, tier_batch(&inputs, COLS)); let plain = workstation::plan_gate(LAYERS); let places = 3 * 4 * 256 * 2; diff --git a/crates/placement/src/accepts.rs b/crates/placement/src/accepts.rs index c06b86c81..f6bd1f8c4 100644 --- a/crates/placement/src/accepts.rs +++ b/crates/placement/src/accepts.rs @@ -508,7 +508,12 @@ pub fn leg_runs(p: Program, [gate, up, down]: [GgmlType; 3]) -> Option { pub fn leg_down_forms(p: Program, model: &crate::placement::ModelTensors) -> kernels::DownForms { use crate::placement::{ExpertPart, Role}; - let RoutedRule::Leg { gate_up, down, pair } = routed_rule(p) else { + let RoutedRule::Leg { + gate_up, + down, + pair, + } = routed_rule(p) + else { return kernels::DownForms::NONE; }; let mut forms = kernels::DownForms::NONE; diff --git a/crates/placement/src/placement.rs b/crates/placement/src/placement.rs index b512fa89d..2c09b7f87 100644 --- a/crates/placement/src/placement.rs +++ b/crates/placement/src/placement.rs @@ -3165,7 +3165,8 @@ impl Fill<'_, '_> { .map(|l| self.model.layer_experts(l)) .collect(); spread(n_l, held, self.eligible, &caps, budget, |n| { - footprint.at(n)? + footprint + .at(n)? .checked_add(scratch(n)?) .ok_or_else(|| card.bytes_past_u64()) }) diff --git a/crates/placement/src/placement/workstation.rs b/crates/placement/src/placement/workstation.rs index 4e1a18893..7df753d30 100644 --- a/crates/placement/src/placement/workstation.rs +++ b/crates/placement/src/placement/workstation.rs @@ -529,12 +529,7 @@ pub const fn walk_a_tile_scratch_bytes(n_embd: u64, ff: u64, n_used: u64, cols: /// columns in the 128-value form ([`leg_bytes`]), in place of the tile path's /// scratch. Selected-form callers use [`tier_leg_batch_bytes_of`]. #[must_use] -pub const fn tier_chunk_scratch_bytes( - n_embd: u64, - ff: u64, - n_used: u64, - chunk: u64, -) -> u64 { +pub const fn tier_chunk_scratch_bytes(n_embd: u64, ff: u64, n_used: u64, chunk: u64) -> u64 { leg_bytes(chunk, n_used, n_embd, ff, DownForms::of(DownAct::Q8Act)) } From efe6c7b63529bbcea43b174b247d2e696cdb9c60 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 05:24:03 +0900 Subject: [PATCH 06/19] Complete the mixed fixture description and remove superseded reserve regression --- crates/gpu/src/card_experts.rs | 2 ++ crates/model/src/arch/glm5next/place.rs | 48 ------------------------- docs/research/flint-attach.md | 7 ++++ 3 files changed, 9 insertions(+), 48 deletions(-) diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index 33e8afca2..eb22b78d7 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -880,6 +880,8 @@ mod tests { top_k: USED as u32, expert_ff: FF as u32, latent: None, + pre_exp_norm: false, + post_exp_norm: None, act: models::Act::SwiGlu { limit: None }, router: Router { score: Score::Sigmoid, diff --git a/crates/model/src/arch/glm5next/place.rs b/crates/model/src/arch/glm5next/place.rs index bca8663f6..99cd5a479 100644 --- a/crates/model/src/arch/glm5next/place.rs +++ b/crates/model/src/arch/glm5next/place.rs @@ -1867,54 +1867,6 @@ mod tests { } } - /// The expert tier's prompt-batch reserve follows the down forms of the file's card layers: - /// its scratch is the card leg's one formula (`kernels::leg_bytes`, which `LegBufs::for_downs` - /// allocates by) over `TIER_CHUNK` tokens in each form a layer's down reads, the staging and - /// host terms beside it moved by no form. A K-quant file, a Q6_K down and a file with no card - /// layer reserve the 128-value form alone; an NVFP4 layer or a Q8_0 down the 32-value form; layers - /// of both, both, the tiles' layers (Q4_K gate and up over a Q5_K down) counted with the rest. - /// Mutants: the reserve of the 128-value form alone; the tiles' layers left out of the forms. - #[test] - fn the_tier_reserve_follows_the_routed_downs() { - use crate::placement::kernels::{DownAct, DownForms, leg_bytes}; - use GgmlType::{IQ3_S, IQ4_XS, NVFP4, Q4_K, Q5_K, Q6_K, Q8_0}; - let q8 = DownForms::of(DownAct::Q8Act); - let b32 = DownForms::of(DownAct::Q8Blocks32); - let base = tier_batch(&inputs()).expect("valid tier batch"); - let hp = inputs().hp; - let leg = |forms| { - leg_bytes( - TIER_CHUNK, - hp.n_used as u64, - hp.n_embd as u64, - hp.expert_ff as u64, - forms, - ) - }; - assert_ne!(leg(q8), leg(b32)); - for (layers, forms) in [ - (vec![], q8), - (vec![[Q4_K, Q4_K, Q5_K]], q8), - (vec![[Q5_K, Q5_K, Q6_K]], q8), - (vec![[IQ3_S, IQ3_S, IQ4_XS]], q8), - (vec![[NVFP4; 3]], b32), - (vec![[Q4_K, Q4_K, Q8_0]], b32), - ( - vec![[IQ3_S, IQ3_S, IQ4_XS], [NVFP4; 3]], - q8.with(DownAct::Q8Blocks32), - ), - ( - vec![[Q4_K, Q4_K, Q5_K], [NVFP4; 3]], - q8.with(DownAct::Q8Blocks32), - ), - ] { - let batch = tier_batch(&routed_types(&layers)).expect("valid selected tier batch"); - assert_eq!(batch.scratch, leg(forms), "{layers:?}"); - assert_eq!(batch.staging, base.staging, "{layers:?}"); - assert_eq!(batch.host, base.host, "{layers:?}"); - } - } - /// A Q5_K gate and up with a Q6_K down takes every expert on a card /// with room for the stack; a Q6_K gate and up takes none. /// PIN(2026-10-22): the down arm admits Q6_K, so the layer is diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 6971fdff8..9ed1cc9e7 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -84,6 +84,13 @@ The wrapper fixture loads each expert part through the selected program's `route compare real step, pass, batch and tier allocations and rows against the common leg. A mismatched gate/up pair remains refused; Solar's admitted Q6_K down is not a rejection case. Tier activation selection remains the family's contract, independently of the program-owned stack admission. + +The 2026-10-26 cutover removes the obsolete always-Q8 tier-reserve regression: +selected mixed-layer and mismatched-gate/up coverage remains in +`tier_scratch_follows_selected_mixed_layers`, with allocation and budget boundaries +covered by the placement and GPU wrapper consumers. The MiniMax planner test retains +its reserve and idle-tier behavior checks rather than comparing descriptor representation +against a different constructor. Host-only projections intentionally have no card totals: fixed reserves remain owned by their machine. Only a dynamic tier-leg descriptor requires the plan's selected card totals. The existing node paging budget regression exercises this distinction without a GPU, including OS reserve and constrained host room. From 3071a9f0d8ed4773692e0c040dece222b90f6bec Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 05:29:37 +0900 Subject: [PATCH 07/19] Drop imports made obsolete by selected reserve regression cutover --- crates/model/src/arch/glm5next/place.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/model/src/arch/glm5next/place.rs b/crates/model/src/arch/glm5next/place.rs index 99cd5a479..c1c6f373a 100644 --- a/crates/model/src/arch/glm5next/place.rs +++ b/crates/model/src/arch/glm5next/place.rs @@ -1480,8 +1480,7 @@ mod tests { use super::super::super::hparams::{Hc, Hparams, Indexer}; use super::super::{ DenseForms, KdaLanes, Kind, KvLayout, NextnInputs, NextnKv, PlaceError, PlanInputs, - TIER_CHUNK, card_leg_bytes, card_leg_forms, prompt_reserve_bytes, row_bytes, - tier_batch, + card_leg_bytes, card_leg_forms, prompt_reserve_bytes, row_bytes, }; use crate::placement::churn::ChurnPool; use crate::placement::workstation::{HostNeed, HostRead}; From c26025b1fb9fad1af399401d27cf709005736ba0 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 06:29:37 +0900 Subject: [PATCH 08/19] Select actual Q8Act tier scratch also for empty tiers --- crates/gpu/src/card_experts.rs | 23 ++++++++++++++++ crates/model/src/arch/mimo2/place.rs | 10 ------- crates/placement/src/placement.rs | 27 +++++++++++-------- crates/placement/src/placement/workstation.rs | 18 ++++++++----- docs/research/flint-attach.md | 22 +++++++++++++-- 5 files changed, 70 insertions(+), 30 deletions(-) diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index eb22b78d7..8946499a6 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -1198,6 +1198,29 @@ mod tests { } assert!(gpu.take_fault().expect("the fault word").is_none()); } + let set = TierSet::new(0..1, EXPERTS, vec![Vec::new()]).expect("an empty tier"); + let shape = FAMILY.shape(N, FF, 2); + let idle = LegTier::new( + &gpu, + &layers(1), + &set, + &Weights::default(), + shape, + FAMILY.program, + tier_act, + ) + .expect("a configured tier need not hold an expert"); + assert_eq!( + idle.block_bytes(), + kernels::leg_bytes( + shape.cap as u64, + USED as u64, + N as u64, + FF as u64, + DownForms::NONE, + ) as usize, + "an empty tier has no down-activation allocation", + ); let triple = [IQ3_S, Q5_K, Q8_0]; let program = Program::Qwen35Body; let w = weights(&gpu, &[triple]); diff --git a/crates/model/src/arch/mimo2/place.rs b/crates/model/src/arch/mimo2/place.rs index 14ad7aa30..1928f6fe2 100644 --- a/crates/model/src/arch/mimo2/place.rs +++ b/crates/model/src/arch/mimo2/place.rs @@ -266,16 +266,6 @@ mod tests { let inputs = PlanInputs::describe(&split).expect("the synthetic header describes"); let _ = std::fs::remove_file(&path); let hp = &inputs.hp; - assert_eq!( - tier_batch(hp, COLS), - workstation::tier_leg_batch_bytes( - hp.n_embd as u64, - hp.expert_ff as u64, - hp.n_used as u64, - TIER_CHUNK, - COLS, - ) - ); let routed = runtime::layer::hosted(&inputs.spec.layers).map_or(0, |run| run.len()); let tiered = workstation::plan_bp(hp.n_trunk, None, tier_batch(hp, COLS)); let plain = workstation::plan_gate(hp.n_trunk); diff --git a/crates/placement/src/placement.rs b/crates/placement/src/placement.rs index 2c09b7f87..32f9c9c8a 100644 --- a/crates/placement/src/placement.rs +++ b/crates/placement/src/placement.rs @@ -6453,13 +6453,20 @@ mod tests { use workstation::{TIER_BATCH_RESERVE, try_tier_leg_batch_bytes_of}; let both = DownForms::of(DownAct::Q8Act).with(DownAct::Q8Blocks32); - let batch = try_tier_leg_batch_bytes_of(256, 256, 1, (2, 3), both).expect("tier bytes"); - let leg = batch.leg.expect("a tier leg descriptor"); + let mixed = try_tier_leg_batch_bytes_of(256, 256, 1, (2, 3), both).expect("tier bytes"); let rule = routed_rule(Program::Qwen35Body); - for (downs, first) in [ - ([Q5_K, Q8_0], DownAct::Q8Act), - ([Q8_0, Q5_K], DownAct::Q8Blocks32), + for (downs, first, batch) in [ + ([Q5_K, Q8_0], DownAct::Q8Act, mixed), + ([Q8_0, Q5_K], DownAct::Q8Blocks32, mixed), + ( + [Q5_K; 2], + DownAct::Q8Act, + workstation::tier_leg_batch_bytes(256, 256, 1, 2, 3), + ), ] { + let scratch = |forms| crate::kernels::leg_bytes(2, 1, 256, 256, forms); + let selected = + DownForms::of(first).with(crate::kernels::down(downs[1]).expect("second down").1); let mut model = legged(&[[Q5_K, Q5_K, downs[0]], [Q5_K, Q5_K, downs[1]]]); model.experts = 1; model.experts_used = 1; @@ -6480,15 +6487,13 @@ mod tests { .map(|t| t.file_bytes / model.experts) .sum::() }; - let one = - batch.staging + leg.bytes(DownForms::of(first)).expect("first scratch") + expert(0); - let two = - batch.staging + leg.bytes(both).expect("mixed scratch") + expert(0) + expert(1); + let one = batch.staging + scratch(DownForms::of(first)) + expert(0); + let two = batch.staging + scratch(selected) + expert(0) + expert(1); for (room, counts, forms) in [ (one - 1, [0, 0], DownForms::NONE), (one, [1, 0], DownForms::of(first)), (two - 1, [1, 0], DownForms::of(first)), - (two, [1, 1], both), + (two, [1, 1], selected), ] { let mut tier = bytes_card("tier", room, 0..0); tier.reserves @@ -6506,7 +6511,7 @@ mod tests { .expect("the joint plan"); assert_eq!(plan.n_l, [0, 0]); assert_eq!(plan.tier_n_l[0], counts, "{downs:?}, {room}"); - let selected = batch.staging + leg.bytes(forms).expect("selected scratch"); + let selected = batch.staging + scratch(forms); assert_eq!( plan.card_reserve(1, TIER_BATCH_RESERVE) .expect("selected reserve"), diff --git a/crates/placement/src/placement/workstation.rs b/crates/placement/src/placement/workstation.rs index 7df753d30..976e3e82c 100644 --- a/crates/placement/src/placement/workstation.rs +++ b/crates/placement/src/placement/workstation.rs @@ -553,7 +553,10 @@ pub const fn tier_batch_bytes(n_embd: u64, ff: u64, n_used: u64, cols: u64) -> T } } -/// [`tier_batch_bytes`] for a fixed common-leg reserve in the 128-value form. +/// [`tier_batch_bytes`] for a candidate-selected common leg in the 128-value form. +/// +/// # Panics +/// Panics on invalid geometry or overflowing byte counts. #[must_use] pub fn tier_leg_batch_bytes( n_embd: u64, @@ -562,12 +565,13 @@ pub fn tier_leg_batch_bytes( chunk: u64, cols: u64, ) -> TierBatchBytes { - TierBatchBytes { - staging: tier_batch_staging_bytes(n_embd, n_used, cols), - scratch: tier_chunk_scratch_bytes(n_embd, ff, n_used, chunk), - host: tier_batch_host_bytes(n_embd, n_used, cols), - leg: None, - } + tier_leg_batch_bytes_of( + n_embd, + ff, + n_used, + (chunk, cols), + DownForms::of(DownAct::Q8Act), + ) } /// The common leg's buffers in `downs`, with the same checked geometry and diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 9ed1cc9e7..6dcce1886 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -2,8 +2,8 @@ Design snapshot: **`e53cabebbb9b16249c81b7c26a026da1670f7d9d`**, requested by [#632](https://github.com/midagedev/bloomery/issues/632). The source-gap observations below describe -that snapshot. The current tree additionally has a **header-only reader**; it still has no Flint -body, app session or serve seat. No GPU, inference, performance or weight-content validation is claimed. +that snapshot. This component tree has a header-only reader and selected common-leg work, +not a Flint body, app session or serve seat. Primitive evidence below is not model admission. ## Host reader status @@ -95,6 +95,24 @@ Host-only projections intentionally have no card totals: fixed reserves remain o Only a dynamic tier-leg descriptor requires the plan's selected card totals. The existing node paging budget regression exercises this distinction without a GPU, including OS reserve and constrained host room. +Follow-up review found a real empty-tier mismatch: the Q8Act-only common constructor still +advertised fixed scratch while `LegTier` selected no down form for zero resident experts. +The expanded public planner regression failed before the fix: reserve24868 versus the +selected allocation20692. `tier_leg_batch_bytes` now produces the same candidate-selected +descriptor as the explicit-form constructor, preserving its infallible signature. +Qwen3/Qwen35, MiMo and Muse therefore price their actual selected union, including empty; +MiniMax, GLM5Next and Solar already use selected constructors. Separate tile/route-based +fixed-reserve paths remain unchanged. No reserve check is weakened and no unused Q8Act +buffer is allocated to hide the mismatch. The same regression passes after the repair. +An old MiMo helper-equality assertion was removed; its actual plan/reserve/refusal checks remain. +The real mixed-wrapper hardware case also checks allocation of a configured empty tier. + +Historical exact-source proof at `3071a9f0d8ed4773692e0c040dece222b90f6bec`: +GPU library418 passed plus10 held passed, verbs1 passed, and all1657 canonical entries +identical to main15778e2c (4bins/8carriers, non-JIT rows equal). +The newly repaired constructor and extended hardware case require their own source-specific +execution; the PR was returned to Draft on the finding rather than relabel the earlier pass. + A host-only public planner consumer exercised the one-byte boundaries before the first resident layer and before adding the other down form, plus descriptor overflow refusal. This proves planning behavior, not device allocation or expert arithmetic. The existing CUDA leg, batch and tier gates still need an From 4d841c94aca30745ed267e0cbd9b2edd71d16540 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:03:30 +0900 Subject: [PATCH 09/19] Record repaired empty-tier consumer hardware proof --- docs/research/flint-attach.md | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 6dcce1886..218196059 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -110,13 +110,19 @@ The real mixed-wrapper hardware case also checks allocation of a configured empt Historical exact-source proof at `3071a9f0d8ed4773692e0c040dece222b90f6bec`: GPU library418 passed plus10 held passed, verbs1 passed, and all1657 canonical entries identical to main15778e2c (4bins/8carriers, non-JIT rows equal). -The newly repaired constructor and extended hardware case require their own source-specific -execution; the PR was returned to Draft on the finding rather than relabel the earlier pass. - -A host-only public planner consumer exercised the one-byte boundaries before the first resident layer -and before adding the other down form, plus descriptor overflow refusal. This proves planning behavior, -not device allocation or expert arithmetic. The existing CUDA leg, batch and tier gates still need an -exact-head NVIDIA run. Flint's `Program` admission and body/app/seat remain absent. +The repair at `c26025b1fb9fad1af399401d27cf709005736ba0` then executed the extended +real CUDA mixed/empty-tier consumer:1 passed,0 failed,0 ignored,427 filtered,24.18s. +Its fresh4bin/8carrier canonical comparison again found all1657entries identical to main15778e2c, +with non-JIT rows equal. Native formatting/Clippy passed (warning3=ratchet3); model557 and +placement202 passed,0 failed,5 model tests ignored,0 filtered. +Request75d4e06e-f3fb-4001-bcc4-e188c169edf5 exited0, credential cleanup unconfirmed; +GPU1 remained active/healthy and was not stopped. The PR returned to review-ready only after +this repair proof. Later documentation-only commits do not count as another hardware run. + +The host-only public planner consumer covers one-byte boundaries before the first resident layer +and before adding another down form, plus descriptor overflow refusal. These and the CUDA consumer +are not a whole-model oracle or every affected model/residency gate; canonical equality does not +waive changed host/allocation consumers. Flint's `Program` admission and body/app/seat remain absent. From 76f4b122d96ccfa7fdc38ae8795e85d92ab8998c Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:48:07 +0900 Subject: [PATCH 10/19] Unify selected tier accounting and checked reserve contracts --- crates/gpu-gates/src/bin/gate_mimo2_serve.rs | 5 +- .../src/bin/gate_musespark_prefill.rs | 2 +- .../gpu-gates/src/bin/shared/mimo2_twocard.rs | 2 +- .../src/bin/shared/minimaxm3_seat.rs | 4 +- .../src/bin/shared/qwen3moe_place.rs | 25 ++++- .../src/bin/shared/serve_seats/mimo2.rs | 4 +- .../src/bin/shared/serve_seats/minimaxm3.rs | 3 +- .../src/bin/shared/serve_seats/musespark.rs | 4 +- crates/gpu-mimo2/src/tier.rs | 5 +- crates/gpu-minimaxm3/src/tier.rs | 5 +- crates/gpu/src/card_experts.rs | 8 +- crates/gpu/src/card_leg.rs | 24 +++-- crates/gpu/src/host/leg_tier.rs | 8 +- crates/model/src/arch/mimo2/place.rs | 9 +- crates/model/src/arch/minimaxm3/place.rs | 40 ++------ crates/model/src/arch/musespark/place.rs | 3 +- crates/model/src/arch/qwen35moe/place.rs | 11 ++- crates/model/src/arch/qwen3moe/place.rs | 3 +- crates/model/src/arch/solar2/place.rs | 64 +++++------- crates/model/tests/mimo2_meta.rs | 2 +- crates/model/tests/musespark_meta.rs | 2 +- crates/model/tests/qwen3_bp_plan.rs | 9 +- crates/placement/src/placement.rs | 91 ++++++++++------- crates/placement/src/placement/workstation.rs | 98 ++++++++++++------- docs/research/flint-attach.md | 32 +++--- 25 files changed, 251 insertions(+), 212 deletions(-) diff --git a/crates/gpu-gates/src/bin/gate_mimo2_serve.rs b/crates/gpu-gates/src/bin/gate_mimo2_serve.rs index 2245c05d3..da2c6271b 100644 --- a/crates/gpu-gates/src/bin/gate_mimo2_serve.rs +++ b/crates/gpu-gates/src/bin/gate_mimo2_serve.rs @@ -892,8 +892,9 @@ mod gate { let mut inputs = PlanInputs::describe(&split)?; inputs.room = (PLAN_ROOM, HostRead::Given); let place = unset_place(cards).on_host()?; - let batch = - (!place.tier_cards().is_empty()).then(|| bloomery_gpu_mimo2::tier::tier_batch(&inputs)); + let batch = (!place.tier_cards().is_empty()) + .then(|| bloomery_gpu_mimo2::tier::tier_batch(&inputs)) + .transpose()?; let machine = place.machine(None, batch)?(inputs.model.layers); let slots = NonZeroUsize::new(usize::try_from(DEFAULT_SLOTS)?).ok_or("no default slot")?; let plan = diff --git a/crates/gpu-gates/src/bin/gate_musespark_prefill.rs b/crates/gpu-gates/src/bin/gate_musespark_prefill.rs index 6674f694b..5137e5828 100644 --- a/crates/gpu-gates/src/bin/gate_musespark_prefill.rs +++ b/crates/gpu-gates/src/bin/gate_musespark_prefill.rs @@ -530,7 +530,7 @@ mod gate { tier_now: Tier, ) -> Result<(Model, bool, Vec), GateError> { let inputs = PlanInputs::read(&open_split(Arch::MuseSpark, RECIPE)?)?; - let mut two = lcpp_e2e::Arm::Bp.machine(hp.n_layer, place::tier_batch(hp))?; + let mut two = lcpp_e2e::Arm::Bp.machine(hp.n_layer, place::tier_batch(hp)?)?; place::stage_placed(&mut two)?; let ctx = u64::try_from(ctx)?; // The stage card alone, the reference's machine: the budget reads its unbudgeted plan, which a plan of the two diff --git a/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs b/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs index 3b8425314..cbf918642 100644 --- a/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs +++ b/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs @@ -111,7 +111,7 @@ pub fn run(path: &Path, levers: &bloomery_levers::Levers) -> Result Result q3::tier_batch(&i.hp), Inputs::Qwen35(i) => q35::leg_tier_batch(&i.spec), - }; + }?; let mut machine = workstation::plan_tiers(layers, *stage, &[*tier], None, batch)?; q3::stage_placed(&mut machine, arena); machine @@ -2099,7 +2099,7 @@ pub(super) mod tests { /// A dense qwen35 file (no router) plans through [`PlaceQ3::qwen35_on`]: its machine counts /// the whole load's ubatch arena, as the load sizes it, not a placed arena, which a file with - /// no router refuses; and the plan answers at the load's context. + /// no router refuses; an expert tier is refused by name through the same consumer. #[allow( dead_code, reason = "`generate_qwen3moe`'s test calls it; every other bin that includes this file leaves it unused" @@ -2137,6 +2137,27 @@ pub(super) mod tests { assert_eq!(workstation::counted_arena_bytes(&q.machine.cards[0]), arena); q.plan(o.ctx, &PlanLevers::default()) .unwrap_or_else(|e| panic!("the dense plan at {}: {e}", o.ctx)); + let tier = DeviceInfo { + ordinal: 1, + name: "NVIDIA RTX 3090".to_owned(), + total_bytes: 25_769_803_776, + free_bytes: 25_000_000_000, + uuid: [2; 16], + pci_bus: "0000:81:00.0".to_owned(), + held_by: None, + integrated: false, + }; + let census = [census.into_iter().next().expect("the stage card"), tier]; + let two = Place::parse("cuda0+1").expect("two card words"); + let refusal = + match PlaceQ3::qwen35_on(&split, two, &census, o, Seqs::ONE, Beside::default()) { + Err(e) => e, + Ok(_) => panic!("a dense model cannot use an expert tier"), + }; + assert!(matches!( + refusal.downcast_ref::(), + Some(PlacementError::Metadata { key, .. }) if key == "Qwen3.6 expert tier" + )); std::fs::remove_file(&path).expect("the header file"); } } diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/mimo2.rs b/crates/gpu-gates/src/bin/shared/serve_seats/mimo2.rs index cf6fd96d0..1204c8373 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/mimo2.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/mimo2.rs @@ -502,7 +502,9 @@ impl Tiered for Mimo { /// The tier card's prompt-batch bytes ([`bloomery_gpu_mimo2::tier::tier_batch`]), under a /// placement with one. fn tier_batch(place: Place, inputs: &PlanInputs) -> Result, GateError> { - Ok((!place.tier_cards().is_empty()).then(|| bloomery_gpu_mimo2::tier::tier_batch(inputs))) + Ok((!place.tier_cards().is_empty()) + .then(|| bloomery_gpu_mimo2::tier::tier_batch(inputs)) + .transpose()?) } fn flags(flag: &str, value: &str, out: &mut Flags) -> Result { diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs b/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs index 096093e07..92f704bf1 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs @@ -506,7 +506,8 @@ impl Tiered for MiniMax { inputs: &PlanInputs, ) -> Result, GateError> { Ok((!place.tier_cards().is_empty()) - .then(|| bloomery_gpu_minimaxm3::tier::tier_batch(inputs))) + .then(|| bloomery_gpu_minimaxm3::tier::tier_batch(inputs)) + .transpose()?) } /// The tags the model's description names for its think span. diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/musespark.rs b/crates/gpu-gates/src/bin/shared/serve_seats/musespark.rs index 0d36b7074..650f28a66 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/musespark.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/musespark.rs @@ -462,7 +462,9 @@ impl Tiered for Muse { place: Place, inputs: &PlanInputs, ) -> Result, GateError> { - Ok((!place.tier_cards().is_empty()).then(|| place::tier_batch(&inputs.hp))) + Ok((!place.tier_cards().is_empty()) + .then(|| place::tier_batch(&inputs.hp)) + .transpose()?) } /// The ring is kept out of the expert budget where the common rule starts diff --git a/crates/gpu-mimo2/src/tier.rs b/crates/gpu-mimo2/src/tier.rs index f0886e36c..eaded5ed9 100644 --- a/crates/gpu-mimo2/src/tier.rs +++ b/crates/gpu-mimo2/src/tier.rs @@ -34,8 +34,8 @@ use gguf::Split; use model::ModelError; use model::arch::mimo2::place::{self, PlanInputs}; use model::moe::HostLayer; -use model::placement::ModelTensors; use model::placement::workstation::TierBatchBytes; +use model::placement::{ModelTensors, PlacementError}; use model::r8file::R8Source; use models::LayerSpec; @@ -56,8 +56,7 @@ pub const CHUNK: usize = place::TIER_CHUNK as usize; /// The tier's prompt-batch bytes the plan reserves on the tier card and the host for the file /// `inputs` describes: [`place::tier_batch`] over blocks of a prompt unit's rows ([`T_MAX`]), the /// columns the load makes the tier's batch legs for. -#[must_use] -pub fn tier_batch(inputs: &PlanInputs) -> TierBatchBytes { +pub fn tier_batch(inputs: &PlanInputs) -> Result { place::tier_batch(&inputs.hp, T_MAX as u64) } diff --git a/crates/gpu-minimaxm3/src/tier.rs b/crates/gpu-minimaxm3/src/tier.rs index ad4f75122..ca916c671 100644 --- a/crates/gpu-minimaxm3/src/tier.rs +++ b/crates/gpu-minimaxm3/src/tier.rs @@ -20,8 +20,8 @@ use gguf::Split; use model::arch::minimaxm3::host as minimax_host; use model::arch::minimaxm3::place::{self, PlanInputs}; use model::moe::HostLayer; -use model::placement::ModelTensors; use model::placement::workstation::TierBatchBytes; +use model::placement::{ModelTensors, PlacementError}; use model::r8file::R8Source; use models::{LayerSpec, Moe}; @@ -38,8 +38,7 @@ pub const CHUNK: usize = place::TIER_CHUNK as usize; /// The tier's prompt-batch bytes the plan reserves on the tier card and the host for the file /// `inputs` describes: [`place::tier_batch`] over blocks of a prompt unit's rows ([`T_MAX`]), the /// columns the load makes the tier's batch legs for. -#[must_use] -pub fn tier_batch(inputs: &PlanInputs) -> TierBatchBytes { +pub fn tier_batch(inputs: &PlanInputs) -> Result { place::tier_batch(inputs, T_MAX as u64) } diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index 8946499a6..13b6365b0 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -46,7 +46,7 @@ use model::placement::accepts::{self, LegArms, Program}; use model::placement::kernels::{self, DownAct, DownForms}; use crate::card_leg::{ - self, CardLeg, LegBufs, LegRun, LegShape, LegStacks, LegSum, RouteBlock, RouteW, + self, CardLeg, LegBufs, LegRun, LegShape, LegStacks, LegSum, RouteBlock, RouteW, STACK_ROLES, }; use crate::card_slots::{CardSlotKernels, Places}; use crate::gemm::Gemm32Kernels; @@ -57,12 +57,6 @@ use crate::ring_gemm::RingGemm; use crate::weights::Weights; use crate::{DeviceTensor, Gpu, GpuError}; -/// The selected program, rather than a union of unrelated launches, admits stacks. -const STACK_ROLES: (&str, &str) = ( - "a resident routed gate or up stack the selected card gate·up arm runs", - "a resident routed down stack the selected card down arm runs", -); - /// What a family fixes of its card experts. #[derive(Clone, Copy)] pub struct CardFamily { diff --git a/crates/gpu/src/card_leg.rs b/crates/gpu/src/card_leg.rs index 1bb6b987c..52b642883 100644 --- a/crates/gpu/src/card_leg.rs +++ b/crates/gpu/src/card_leg.rs @@ -514,14 +514,12 @@ struct LegStack<'a> { } impl<'a> LegStack<'a> { - /// The resident stack `name`: a `KQuant` stack of a type `runs` admits in - /// the stack's role, or an NVFP4 stack in its planes where `planes` - /// admits NVFP4; `role` is the refusal's words. + /// The resident stack `name` of a type `runs` admits in its role, + /// including plane-backed NVFP4 when admitted; `role` names a refusal. fn of( w: &'a Weights, name: &'a str, runs: fn(GgmlType) -> bool, - planes: fn(GgmlType) -> bool, role: &'static str, ) -> Result, GpuError> { match w.get(name) { @@ -537,7 +535,7 @@ impl<'a> LegStack<'a> { w: StackW::Q5_0(w), k: *k, }), - Some(DevWeight::Nvfp4Stack { k, .. }) if planes(GgmlType::NVFP4) => Ok(LegStack { + Some(DevWeight::Nvfp4Stack { k, .. }) if runs(GgmlType::NVFP4) => Ok(LegStack { name, ty: GgmlType::NVFP4, w: StackW::Nvfp4(w.nvfp4_stack(WHAT, name)?), @@ -609,6 +607,12 @@ impl<'a> LegStack<'a> { const GATE_UP: &str = "a resident routed gate or up stack of a type the card leg runs (`kernels::card_gate_up_runs`)"; +/// The selected program admits the same resident stack roles on card and tier. +pub(crate) const STACK_ROLES: (&str, &str) = ( + "a resident routed gate or up stack the selected program's gate·up arm runs", + "a resident routed down stack the selected program's down arm runs", +); + /// A layer's three resident routed stacks, the gate and up of one type. #[derive(Clone, Copy)] pub struct LegStacks<'a> { @@ -670,9 +674,9 @@ impl<'a> LegStacks<'a> { ) -> Result, GpuError> { let [gate, up, down] = names; let (gate, up, down) = ( - LegStack::of(w, gate, arms.gate_up, arms.gate_up, roles.0)?, - LegStack::of(w, up, arms.gate_up, arms.gate_up, roles.0)?, - LegStack::of(w, down, arms.down, arms.down, roles.1)?, + LegStack::of(w, gate, arms.gate_up, roles.0)?, + LegStack::of(w, up, arms.gate_up, roles.0)?, + LegStack::of(w, down, arms.down, roles.1)?, ); if up.ty != gate.ty { return Err(GpuError::Tensor { @@ -1492,8 +1496,8 @@ impl<'a> Relu2Stacks<'a> { const DOWN: &str = "a resident routed down stack of a 32-value-block type the card leg runs (`kernels::card_sel32_runs`)"; let [up, down] = names; Ok(Relu2Stacks { - up: LegStack::of(w, up, kernels::card_sel32_runs, |_| false, UP)?, - down: LegStack::of(w, down, kernels::card_sel32_runs, |_| false, DOWN)?, + up: LegStack::of(w, up, kernels::card_sel32_runs, UP)?, + down: LegStack::of(w, down, kernels::card_sel32_runs, DOWN)?, }) } diff --git a/crates/gpu/src/host/leg_tier.rs b/crates/gpu/src/host/leg_tier.rs index 2eecc718b..c35afc97d 100644 --- a/crates/gpu/src/host/leg_tier.rs +++ b/crates/gpu/src/host/leg_tier.rs @@ -40,7 +40,7 @@ use super::run::{HostRun, HostWidths}; use super::tier::{ TierAct, TierBlock, TierCard, TierExperts, TierInput, TierIo, TierOpen, TierSet, TierShape, }; -use crate::card_leg::{CardLeg, LegBufs, LegRun, LegShape, LegStacks}; +use crate::card_leg::{CardLeg, LegBufs, LegRun, LegShape, LegStacks, STACK_ROLES}; use crate::hybrid::{Hybrid, SlotMap}; use crate::kquant::Act; use crate::model::CardBytes; @@ -53,12 +53,6 @@ const WHAT: &str = "LegTier"; /// What the stage card's side of the tier layers names in its errors. const SIDE: &str = "TierSide"; -/// The selected program's launches admit the same stacks on both cards. -const STACK_ROLES: (&str, &str) = ( - "a resident routed gate or up stack the selected tier gate·up arm runs", - "a resident routed down stack the selected tier down arm runs", -); - /// The tier the stage card's tier entries serve: they hand one tier its /// image and join one tier's rows, so a load holds one tier card /// ([`crate::host::refuse_tier_count`]). diff --git a/crates/model/src/arch/mimo2/place.rs b/crates/model/src/arch/mimo2/place.rs index 1928f6fe2..03e4345ec 100644 --- a/crates/model/src/arch/mimo2/place.rs +++ b/crates/model/src/arch/mimo2/place.rs @@ -119,8 +119,7 @@ pub const TIER_CHUNK: u64 = 16; /// plan's rule carries reads (the rule's down arm is the leg's `Q8Act` downs), and the host's rows /// and places ([`workstation::tier_leg_batch_bytes`]). The plan reserves them on the tier card and /// the host, and the load checks its allocations against them. -#[must_use] -pub fn tier_batch(hp: &Hparams, cols: u64) -> TierBatchBytes { +pub fn tier_batch(hp: &Hparams, cols: u64) -> Result { workstation::tier_leg_batch_bytes( hp.n_embd as u64, hp.expert_ff as u64, @@ -267,7 +266,11 @@ mod tests { let _ = std::fs::remove_file(&path); let hp = &inputs.hp; let routed = runtime::layer::hosted(&inputs.spec.layers).map_or(0, |run| run.len()); - let tiered = workstation::plan_bp(hp.n_trunk, None, tier_batch(hp, COLS)); + let tiered = workstation::plan_bp( + hp.n_trunk, + None, + tier_batch(hp, COLS).expect("valid tier geometry"), + ); let plain = workstation::plan_gate(hp.n_trunk); assert_eq!( inputs.prompt_batch_bytes(&plain).expect("a plain machine"), diff --git a/crates/model/src/arch/minimaxm3/place.rs b/crates/model/src/arch/minimaxm3/place.rs index 309200f48..e8528fb48 100644 --- a/crates/model/src/arch/minimaxm3/place.rs +++ b/crates/model/src/arch/minimaxm3/place.rs @@ -22,10 +22,10 @@ use crate::arch::names::routed_stacks; use crate::arch::place_hybrid::{ self, ring_pass_bytes, route_bytes, stream_ring_bytes, tier_places_bytes, wide_bytes, }; -use crate::placement::accepts::planes_leg_down; +use crate::placement::PlacementError; +use crate::placement::accepts::{self, Program}; use crate::placement::kernels::{self, DownAct, DownForms}; use crate::placement::workstation::{self, TierBatchBytes}; -use crate::placement::{PlacementError, Role}; pub use place_hybrid::{KvLayout, PlaceError}; @@ -134,41 +134,17 @@ pub const TIER_CHUNK: u64 = 16; /// (an NVFP4 down's 32-value blocks, a K- or i-quant down's 128-value form) and the host's rows /// and places. The plan reserves them on the tier card and the host, and the load checks its /// allocations against them. -#[must_use] -pub fn tier_batch(inputs: &PlanInputs, cols: u64) -> TierBatchBytes { +pub fn tier_batch(inputs: &PlanInputs, cols: u64) -> Result { let hp = &inputs.hp; - workstation::tier_leg_batch_bytes_of( + workstation::try_tier_leg_batch_bytes_of( hp.n_embd as u64, hp.experts.ff as u64, hp.experts.n_used as u64, (TIER_CHUNK, cols), - leg_downs(inputs), + accepts::leg_down_forms(Program::MiniMaxM3Body, &inputs.model), ) } -/// The forms the card leg's routed downs of `inputs` read: each routed layer's down stack whose -/// type the leg runs ([`planes_leg_down`]), by its kernel table row ([`kernels::down`]); the -/// 128-value form on a file with none. -fn leg_downs(inputs: &PlanInputs) -> DownForms { - let downs = (0..inputs.model.layers) - .map(|l| routed_stacks(l)[2].clone()) - .filter_map(|name| { - inputs - .model - .tensors - .iter() - .find(|t| t.role == Role::RoutedExperts && t.name == name) - }) - .filter(|t| planes_leg_down(t.ty)) - .filter_map(|t| kernels::down(t.ty)) - .fold(DownForms::NONE, |forms, (_, act)| forms.with(act)); - if downs == DownForms::NONE { - DownForms::of(DownAct::Q8Act) - } else { - downs - } -} - /// The slots of each half of the expert stream's ring a streaming plan keeps room for /// ([`stream_bytes`]) [derived: at the ring pass's route GEMMs the stream rule's crossover is about /// 18 columns of a walk, and a walk of the default group (two units of [`PROMPT_ROWS`] rows, four @@ -425,7 +401,11 @@ mod tests { fn a_tier_card_is_planned_with_the_leg_batch() { const COLS: u64 = 3 * TIER_CHUNK; let inputs = described("mm3-place-tier"); - let tiered = workstation::plan_bp(LAYERS, None, tier_batch(&inputs, COLS)); + let tiered = workstation::plan_bp( + LAYERS, + None, + tier_batch(&inputs, COLS).expect("valid tier geometry"), + ); let plain = workstation::plan_gate(LAYERS); let places = 3 * 4 * 256 * 2; assert_eq!(inputs.tier_reserve, Some(places)); diff --git a/crates/model/src/arch/musespark/place.rs b/crates/model/src/arch/musespark/place.rs index e47ddf052..f0900aced 100644 --- a/crates/model/src/arch/musespark/place.rs +++ b/crates/model/src/arch/musespark/place.rs @@ -76,8 +76,7 @@ pub const fn pass_rows_max(n_used: usize) -> usize { /// placed pass's rows ([`pass_rows_max`]): the experts run in the latent, so /// the tier's rows and its staging are the latent's width, through the card /// leg in chunks of [`TIER_CHUNK`]. -#[must_use] -pub fn tier_batch(hp: &Hparams) -> TierBatchBytes { +pub fn tier_batch(hp: &Hparams) -> Result { let ff = hp.expert_width(); workstation::tier_leg_batch_bytes( hp.latent as u64, diff --git a/crates/model/src/arch/qwen35moe/place.rs b/crates/model/src/arch/qwen35moe/place.rs index 2f60c4201..7433ccbc3 100644 --- a/crates/model/src/arch/qwen35moe/place.rs +++ b/crates/model/src/arch/qwen35moe/place.rs @@ -1498,11 +1498,12 @@ pub const LEG_TIER_CHUNK: u64 = 8; /// card's staging, one chunk of the leg's scratch and the host's rows, for /// blocks of the host union's columns. [`tier_batch`] is the qwen4exp plan's /// figure. -#[must_use] -pub fn leg_tier_batch(spec: &ModelSpec) -> TierBatchBytes { - let (expert_ff, n_used) = spec - .routed() - .map_or((0, 0), |m| (u64::from(m.expert_ff), u64::from(m.top_k))); +pub fn leg_tier_batch(spec: &ModelSpec) -> Result { + let moe = spec.routed().ok_or_else(|| PlacementError::Metadata { + key: "Qwen3.6 expert tier".into(), + detail: "a dense model has no routed experts for an expert tier".into(), + })?; + let (expert_ff, n_used) = (u64::from(moe.expert_ff), u64::from(moe.top_k)); workstation::tier_leg_batch_bytes( u64::from(spec.hidden), expert_ff, diff --git a/crates/model/src/arch/qwen3moe/place.rs b/crates/model/src/arch/qwen3moe/place.rs index 3c71a02d2..8afc5c3ed 100644 --- a/crates/model/src/arch/qwen3moe/place.rs +++ b/crates/model/src/arch/qwen3moe/place.rs @@ -45,8 +45,7 @@ pub const TIER_CHUNK: u64 = 8; /// The expert tier's prompt-batch bytes ([`TierBatchBytes`]) for blocks of /// the host union's columns ([`crate::moe::UNION_MAX_COLS`]): the tier card's /// staging, one chunk of the card leg's scratch and the host's rows. -#[must_use] -pub fn tier_batch(hp: &Hparams) -> TierBatchBytes { +pub fn tier_batch(hp: &Hparams) -> Result { workstation::tier_leg_batch_bytes( hp.n_embd as u64, hp.experts.ff as u64, diff --git a/crates/model/src/arch/solar2/place.rs b/crates/model/src/arch/solar2/place.rs index 8aaf7f92a..8c011861e 100644 --- a/crates/model/src/arch/solar2/place.rs +++ b/crates/model/src/arch/solar2/place.rs @@ -176,7 +176,7 @@ mod tests { use super::super::hparams::tests::{LAYERS, keys, tensors}; use super::{PROMPT_ROWS, PlaceError, PlanInputs, prompt_reserve_bytes, tier_batch}; - use crate::arch::synthetic::{V, header_shaped}; + use crate::arch::synthetic::{Ty, V, header_shaped, header_typed}; use crate::placement::{Device, PlacementError, PlanLevers, Violation, workstation}; /// The small header's file, described; each test names its own file (the tests run in @@ -319,9 +319,6 @@ mod tests { /// at the allocation). #[test] fn a_host_only_mixture_keeps_the_exact_prompt_reserve_boundary() { - use crate::placement::{ExpertPart, Role}; - use gguf::GgmlType::{Q5_K, Q8_0}; - let mut kv = keys(); for (key, value) in &mut kv { if matches!(*key, "embedding_length" | "expert_feed_forward_length") { @@ -367,47 +364,28 @@ mod tests { ("tokenizer.ggml.pre", V::Str("solar-open")), ("tokenizer.chat_template", V::Str("{{ messages }}")), ]; - let path = header_shaped("so2-place-mixed-reserve", "solar-open2", &kv, &global, &ts); + let ts: Vec<_> = ts + .into_iter() + .map(|(name, dims)| { + let ty = if name == "blk.0.ffn_down_exps.weight" { + Ty::Q8_0 + } else if matches!( + name.rsplit('.').nth(1), + Some("ffn_gate_exps" | "ffn_up_exps" | "ffn_down_exps") + ) { + Ty::Q4K + } else { + Ty::F32 + }; + (name, dims, ty) + }) + .collect(); + let path = header_typed("so2-place-mixed-reserve", "solar-open2", &kv, &global, &ts); let split = Split::open(&path).expect("the mixed fixture opens"); - let mut inputs = PlanInputs::describe(&split).expect("valid routed geometry"); + let inputs = PlanInputs::describe(&split).expect("valid routed geometry"); std::fs::remove_file(path).expect("remove the header fixture"); - for t in inputs - .model - .tensors - .iter_mut() - .filter(|t| t.role == Role::RoutedExperts) - { - t.ty = Q5_K; - t.file_bytes = t.dims.iter().product::() / t.ty.blck_size().unwrap() - * t.ty.type_size().unwrap(); - } - let admitted = prompt_reserve_bytes(&inputs.hp, &inputs.spec, &inputs.model, PROMPT_ROWS) - .expect("the admitted prompt reserve"); - let admitted_tier = tier_batch(&inputs).expect("the admitted tier reserve"); - let down = inputs - .model - .tensors - .iter_mut() - .find(|t| { - t.role == Role::RoutedExperts - && t.layer == Some(0) - && ExpertPart::of_name(&t.name) == Some(ExpertPart::Down) - }) - .expect("layer zero's down"); - down.ty = Q8_0; - down.file_bytes = down.dims.iter().product::() / down.ty.blck_size().unwrap() - * down.ty.type_size().unwrap(); let reserve = prompt_reserve_bytes(&inputs.hp, &inputs.spec, &inputs.model, PROMPT_ROWS) .expect("prompt reserve"); - assert_eq!( - reserve, admitted, - "the host-only Q8_0 down adds no card form" - ); - assert_eq!( - tier_batch(&inputs).expect("mixed tier reserve"), - admitted_tier - ); - inputs.prompt_reserve = reserve; let ample = workstation::plan_gate(LAYERS); let placed = inputs .plan(&le, 64, &PlanLevers::default()) @@ -457,5 +435,9 @@ mod tests { .plan(&machine, 64, &levers) .expect("the card holds the terms and the reserve"); assert!(plan.violations().is_empty()); + assert!( + plan.n_l.iter().all(|&n| n == 0), + "the exact dense-and-prompt budget leaves no expert room" + ); } } diff --git a/crates/model/tests/mimo2_meta.rs b/crates/model/tests/mimo2_meta.rs index 64ecda04a..a48c53896 100644 --- a/crates/model/tests/mimo2_meta.rs +++ b/crates/model/tests/mimo2_meta.rs @@ -401,7 +401,7 @@ fn hw_mimo2_plan() { let tiered = workstation::plan_bp( layers, None, - model::arch::mimo2::place::tier_batch(&inputs.hp, 512), + model::arch::mimo2::place::tier_batch(&inputs.hp, 512).expect("valid tier geometry"), ); let bp = inputs .plan(&tiered, ctx, &PlanLevers::default()) diff --git a/crates/model/tests/musespark_meta.rs b/crates/model/tests/musespark_meta.rs index d3cabdd9c..079fc4c19 100644 --- a/crates/model/tests/musespark_meta.rs +++ b/crates/model/tests/musespark_meta.rs @@ -368,7 +368,7 @@ fn hw_musespark_plan() { workstation::A6000, &[workstation::RTX_3090], None, - place::tier_batch(&hp), + place::tier_batch(&hp).expect("valid tier geometry"), ) .unwrap_or_else(|e| panic!("the tiered machine: {e}")); place::stage_placed(&mut m).unwrap_or_else(|e| panic!("the tiered machine's stage: {e}")); diff --git a/crates/model/tests/qwen3_bp_plan.rs b/crates/model/tests/qwen3_bp_plan.rs index ff86bc0f0..64175fc36 100644 --- a/crates/model/tests/qwen3_bp_plan.rs +++ b/crates/model/tests/qwen3_bp_plan.rs @@ -522,7 +522,7 @@ fn qwen36_inputs() -> q35::PlanInputs { fn qwen36_tier(stage: CardSpec, tier: CardSpec) -> Result<(u64, u64, u64), Place35Error> { let inputs = qwen36_inputs(); - let machine = tiered(Q36_LAYERS, stage, tier, q35::leg_tier_batch(&inputs.spec)); + let machine = tiered(Q36_LAYERS, stage, tier, q35::leg_tier_batch(&inputs.spec)?); let plan = inputs.plan_rule( &machine, CTX, @@ -537,7 +537,12 @@ fn qwen36_tier(stage: CardSpec, tier: CardSpec) -> Result<(u64, u64, u64), Place #[test] fn qwen3_30b_q8_bp_tier_experts() { let inputs = qwen3_inputs(); - let machine = tiered(Q3_LAYERS, A6000, RTX_3090, q3::tier_batch(&inputs.hp)); + let machine = tiered( + Q3_LAYERS, + A6000, + RTX_3090, + q3::tier_batch(&inputs.hp).expect("valid tier geometry"), + ); let plan = inputs .plan(&machine, CTX, &PlanLevers::default()) .expect("the A6000 stage plans a tier"); diff --git a/crates/placement/src/placement.rs b/crates/placement/src/placement.rs index 32f9c9c8a..2d57f3239 100644 --- a/crates/placement/src/placement.rs +++ b/crates/placement/src/placement.rs @@ -287,20 +287,14 @@ impl Card { let Some(leg) = self.tier_leg else { return Ok(None); }; - let forms = model - .tensors - .iter() - .filter(|t| { - t.role == Role::RoutedExperts - && t.layer - .is_some_and(|l| counts.get(l).is_some_and(|&n| n > 0)) - && ExpertPart::of_name(&t.name) == Some(ExpertPart::Down) - }) - .filter_map(|t| crate::kernels::down(t.ty)) - .fold(crate::kernels::DownForms::NONE, |forms, (_, act)| { - forms.with(act) - }); - Ok(Some((leg.bytes(leg.downs)?, leg.bytes(forms)?))) + Ok(Some(( + leg.bytes(leg.downs)?, + selected_leg_scratch( + leg, + counts, + routed_down_acts(model).map(|(l, act)| (l, crate::kernels::DownForms::of(act))), + )?, + ))) } fn selected_reserve_bytes( @@ -372,6 +366,41 @@ impl Card { } } +/// Routed down forms with their layers, before a placement selects any experts. +fn routed_down_acts( + model: &ModelTensors, +) -> impl Iterator + '_ { + model.tensors.iter().filter_map(|t| { + if t.role != Role::RoutedExperts || ExpertPart::of_name(&t.name) != Some(ExpertPart::Down) { + return None; + } + Some((t.layer?, crate::kernels::down(t.ty)?.1)) + }) +} + +/// Price the actual selected union, from tensors or cached per-layer forms. +/// Empty counts and layers with no down keep the descriptor's empty-form scratch. +fn selected_leg_scratch( + leg: workstation::TierLegScratch, + counts: &[u64], + downs: impl IntoIterator, +) -> Result { + use crate::kernels::{DownAct, DownForms}; + + let forms = downs + .into_iter() + .filter(|(l, _)| counts.get(*l).is_some_and(|&n| n > 0)) + .fold(DownForms::NONE, |mut forms, (_, down)| { + for act in [DownAct::Q8Act, DownAct::Q8Blocks32] { + if down.holds(act) { + forms = forms.with(act); + } + } + forms + }); + leg.bytes(forms) +} + /// The host tier: its usable bytes and what is set aside before any tensor. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Host { @@ -3109,32 +3138,23 @@ impl Fill<'_, '_> { .routed .unwrap_or(RoutedRule::PerType(CardFormat::of_routed)); let uploads = card_uploads(self.model, self.c, self.rows, self.eligible, format)?; - let mut down_acts = if card.tier_leg.is_some() { - vec![None; self.model.layers] + // Only tier legs prepare the per-layer forms. Candidates reuse these + // rather than scanning tensors, including when a layer has several downs. + let mut down_forms = if card.tier_leg.is_some() { + vec![crate::kernels::DownForms::NONE; self.model.layers] } else { Vec::new() }; if card.tier_leg.is_some() { - for t in &self.model.tensors { - if t.role == Role::RoutedExperts - && ExpertPart::of_name(&t.name) == Some(ExpertPart::Down) - && let Some(l) = t.layer - && let Some((_, act)) = crate::kernels::down(t.ty) - { - down_acts[l] = Some(act); + for (l, act) in routed_down_acts(self.model) { + if let Some(forms) = down_forms.get_mut(l) { + *forms = forms.with(act); } } } let scratch = |counts: &[u64]| { card.tier_leg.map_or(Ok(0), |leg| { - let forms = counts - .iter() - .zip(&down_acts) - .filter_map(|(&n, &act)| (n > 0).then_some(act).flatten()) - .fold(crate::kernels::DownForms::NONE, |forms, act| { - forms.with(act) - }); - leg.bytes(forms) + selected_leg_scratch(leg, counts, down_forms.iter().copied().enumerate()) }) }; let bound = card @@ -3142,7 +3162,12 @@ impl Fill<'_, '_> { .map(|leg| leg.bytes(leg.downs)) .transpose()? .unwrap_or(0); - let reserves = card.selected_reserve_bytes(self.model, n_l)?; + let selected = scratch(n_l)?; + let reserves = card + .reserve_bytes() + .and_then(|fixed| fixed.checked_sub(bound)) + .and_then(|fixed| fixed.checked_add(selected)) + .ok_or_else(|| card.bytes_past_u64())?; let footprint = Footprint::new(card.granule_bytes, &uploads); let budget = i128::from(capped(card, self.card_budget)) - i128::from(self.kv) @@ -6461,7 +6486,7 @@ mod tests { ( [Q5_K; 2], DownAct::Q8Act, - workstation::tier_leg_batch_bytes(256, 256, 1, 2, 3), + workstation::tier_leg_batch_bytes(256, 256, 1, 2, 3).expect("valid tier bytes"), ), ] { let scratch = |forms| crate::kernels::leg_bytes(2, 1, 256, 256, forms); diff --git a/crates/placement/src/placement/workstation.rs b/crates/placement/src/placement/workstation.rs index 976e3e82c..5ab150537 100644 --- a/crates/placement/src/placement/workstation.rs +++ b/crates/placement/src/placement/workstation.rs @@ -476,8 +476,38 @@ const TILE_COLS: u64 = 8; /// activations, the tier places, their q8_1 form, the down outputs by slot. #[must_use] pub const fn tier_batch_staging_bytes(n_embd: u64, n_used: u64, cols: u64) -> u64 { - let slots = cols * n_used; - 4 * cols * n_embd + 4 * slots + q8_act_bytes(cols, n_embd) + 4 * slots * n_embd + match try_tier_batch_staging_bytes(n_embd, n_used, cols) { + Some(bytes) => bytes, + None => panic!("tier_batch_staging_bytes: staging bytes overflow u64"), + } +} + +const fn try_tier_batch_staging_bytes(n_embd: u64, n_used: u64, cols: u64) -> Option { + let Some(slots) = cols.checked_mul(n_used) else { + return None; + }; + let Some(places) = 4u64.checked_mul(slots) else { + return None; + }; + let Some(down) = places.checked_mul(n_embd) else { + return None; + }; + let Some(rows) = 4u64.checked_mul(cols) else { + return None; + }; + let Some(rows) = rows.checked_mul(n_embd) else { + return None; + }; + let Some(bytes) = rows.checked_add(places) else { + return None; + }; + let Some(q8) = try_q8_act_bytes(cols, n_embd) else { + return None; + }; + let Some(bytes) = bytes.checked_add(q8) else { + return None; + }; + bytes.checked_add(down) } /// The tile path's scratch for blocks of up to `slots` slots of rows of @@ -527,7 +557,7 @@ pub const fn walk_a_tile_scratch_bytes(n_embd: u64, ff: u64, n_used: u64, cols: /// token, through experts of `ff`, cut in chunks of `chunk` tokens: one run of /// the card leg's buffers (`bloomery_gpu::card_leg::LegBufs`) with its slots' /// columns in the 128-value form ([`leg_bytes`]), in place of the tile path's -/// scratch. Selected-form callers use [`tier_leg_batch_bytes_of`]. +/// scratch. Selected-form callers use [`try_tier_leg_batch_bytes_of`]. #[must_use] pub const fn tier_chunk_scratch_bytes(n_embd: u64, ff: u64, n_used: u64, chunk: u64) -> u64 { leg_bytes(chunk, n_used, n_embd, ff, DownForms::of(DownAct::Q8Act)) @@ -537,8 +567,26 @@ pub const fn tier_chunk_scratch_bytes(n_embd: u64, ff: u64, n_used: u64, chunk: /// rows the tier hands back and the places it reads. #[must_use] pub const fn tier_batch_host_bytes(n_embd: u64, n_used: u64, cols: u64) -> u64 { - let slots = cols * n_used; - 2 * (4 * slots * n_embd + 4 * slots) + match try_tier_batch_host_bytes(n_embd, n_used, cols) { + Some(bytes) => bytes, + None => panic!("tier_batch_host_bytes: host bytes overflow u64"), + } +} + +const fn try_tier_batch_host_bytes(n_embd: u64, n_used: u64, cols: u64) -> Option { + let Some(slots) = cols.checked_mul(n_used) else { + return None; + }; + let Some(places) = 4u64.checked_mul(slots) else { + return None; + }; + let Some(down) = places.checked_mul(n_embd) else { + return None; + }; + let Some(rows_and_places) = down.checked_add(places) else { + return None; + }; + 2u64.checked_mul(rows_and_places) } /// The expert tier's prompt-batch bytes for blocks of up to `cols` tokens of @@ -555,17 +603,15 @@ pub const fn tier_batch_bytes(n_embd: u64, ff: u64, n_used: u64, cols: u64) -> T /// [`tier_batch_bytes`] for a candidate-selected common leg in the 128-value form. /// -/// # Panics -/// Panics on invalid geometry or overflowing byte counts. -#[must_use] +/// Refuses invalid geometry or overflowing byte counts before placement. pub fn tier_leg_batch_bytes( n_embd: u64, ff: u64, n_used: u64, chunk: u64, cols: u64, -) -> TierBatchBytes { - tier_leg_batch_bytes_of( +) -> Result { + try_tier_leg_batch_bytes_of( n_embd, ff, n_used, @@ -574,23 +620,6 @@ pub fn tier_leg_batch_bytes( ) } -/// The common leg's buffers in `downs`, with the same checked geometry and -/// candidate-dependent reserve as [`try_tier_leg_batch_bytes_of`]. -/// -/// # Panics -/// Panics on invalid geometry or overflowing byte counts. -#[must_use] -pub fn tier_leg_batch_bytes_of( - n_embd: u64, - ff: u64, - n_used: u64, - shape: (u64, u64), - downs: DownForms, -) -> TierBatchBytes { - try_tier_leg_batch_bytes_of(n_embd, ff, n_used, shape, downs) - .expect("valid common leg batch geometry and byte counts") -} - /// The common leg's staging, host rows and selected-form scratch, refusing /// invalid geometry or overflowing byte counts before placement. pub fn try_tier_leg_batch_bytes_of( @@ -612,17 +641,10 @@ pub fn try_tier_leg_batch_bytes_of( if cols == 0 { return None; } - let slots = cols.checked_mul(n_used)?; - let places = 4u64.checked_mul(slots)?; - let down = places.checked_mul(n_embd)?; - let staging = 4u64 - .checked_mul(cols)? - .checked_mul(n_embd)? - .checked_add(places)? - .checked_add(try_q8_act_bytes(cols, n_embd)?)? - .checked_add(down)?; - let host = 2u64.checked_mul(down.checked_add(places)?)?; - Some((staging, host)) + Some(( + try_tier_batch_staging_bytes(n_embd, n_used, cols)?, + try_tier_batch_host_bytes(n_embd, n_used, cols)?, + )) }; let (staging, host) = counted().ok_or_else(|| PlacementError::Metadata { key: "tier leg batch".into(), diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 218196059..013052334 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -95,16 +95,25 @@ Host-only projections intentionally have no card totals: fixed reserves remain o Only a dynamic tier-leg descriptor requires the plan's selected card totals. The existing node paging budget regression exercises this distinction without a GPU, including OS reserve and constrained host room. -Follow-up review found a real empty-tier mismatch: the Q8Act-only common constructor still -advertised fixed scratch while `LegTier` selected no down form for zero resident experts. -The expanded public planner regression failed before the fix: reserve24868 versus the -selected allocation20692. `tier_leg_batch_bytes` now produces the same candidate-selected -descriptor as the explicit-form constructor, preserving its infallible signature. -Qwen3/Qwen35, MiMo and Muse therefore price their actual selected union, including empty; -MiniMax, GLM5Next and Solar already use selected constructors. Separate tile/route-based -fixed-reserve paths remain unchanged. No reserve check is weakened and no unused Q8Act -buffer is allocated to hide the mismatch. The same regression passes after the repair. -An old MiMo helper-equality assertion was removed; its actual plan/reserve/refusal checks remain. +`tier_leg_batch_bytes` and the explicit-form `try_tier_leg_batch_bytes_of` return +`Result` and construct candidate-selected descriptors. +An empty tier selects no down form, matching `LegTier`'s actual allocation instead of +advertising unused Q8Act scratch. Qwen3/Qwen35, MiMo and Muse use this checked path. +MiniMax obtains its forms from `accepts::leg_down_forms` under its selected program, +as GLM5Next and Solar do; no fallback format can admit a mismatched triple. +The checked constructors and non-leg paths share staging and host byte formulas. +Malformed geometry and overflow are refused rather than panicking; a dense Qwen3.6 +request for an expert tier is a named metadata refusal, not zero expert geometry. +Separate tile/route-based fixed-reserve paths remain unchanged. + +The planner's selected scratch calculation has one union/byte rule. Fill prepares +per-layer forms once for dynamic tier-leg cards, including multiple forms in one +layer, and reuses them with its cached footprint instead of rescanning tensors for +each candidate. Fixed-reserve cards need no selected-form cache. + +The Solar budget regression starts from a typed mixed GGUF description and exercises +the planner's one-byte refusal/success boundary without overwriting its described +prompt reserve. The exact dense-and-prompt budget leaves no room for expert weights. The real mixed-wrapper hardware case also checks allocation of a configured empty tier. Historical exact-source proof at `3071a9f0d8ed4773692e0c040dece222b90f6bec`: @@ -115,9 +124,6 @@ real CUDA mixed/empty-tier consumer:1 passed,0 failed,0 ignored,427 filtered,24. Its fresh4bin/8carrier canonical comparison again found all1657entries identical to main15778e2c, with non-JIT rows equal. Native formatting/Clippy passed (warning3=ratchet3); model557 and placement202 passed,0 failed,5 model tests ignored,0 filtered. -Request75d4e06e-f3fb-4001-bcc4-e188c169edf5 exited0, credential cleanup unconfirmed; -GPU1 remained active/healthy and was not stopped. The PR returned to review-ready only after -this repair proof. Later documentation-only commits do not count as another hardware run. The host-only public planner consumer covers one-byte boundaries before the first resident layer and before adding another down form, plus descriptor overflow refusal. These and the CUDA consumer From 5bee29875664d05cfd6cd9ddbdf49ffc50c7a684 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:52:05 +0900 Subject: [PATCH 11/19] Bound Solar reserve regression from a host-only reference plan --- crates/model/src/arch/solar2/place.rs | 51 +++++++++++---------------- 1 file changed, 20 insertions(+), 31 deletions(-) diff --git a/crates/model/src/arch/solar2/place.rs b/crates/model/src/arch/solar2/place.rs index 8c011861e..4de9ecd33 100644 --- a/crates/model/src/arch/solar2/place.rs +++ b/crates/model/src/arch/solar2/place.rs @@ -177,7 +177,7 @@ mod tests { use super::super::hparams::tests::{LAYERS, keys, tensors}; use super::{PROMPT_ROWS, PlaceError, PlanInputs, prompt_reserve_bytes, tier_batch}; use crate::arch::synthetic::{Ty, V, header_shaped, header_typed}; - use crate::placement::{Device, PlacementError, PlanLevers, Violation, workstation}; + use crate::placement::{Device, PlacementError, PlanLevers, workstation}; /// The small header's file, described; each test names its own file (the tests run in /// threads of one process). @@ -400,36 +400,25 @@ mod tests { ); let levers = PlanLevers::default(); let mut machine = workstation::plan_gate(LAYERS); - machine.cards[0].usable_bytes = machine.cards[0].margin_bytes; - let short = loop { - match inputs.plan(&machine, 64, &levers) { - Err(PlaceError::Broken(v)) => { - let over = v - .iter() - .find_map(|v| match v { - Violation::CardOver { total, limit, .. } => Some(total - limit), - _ => None, - }) - .unwrap_or_else(|| panic!("{v:?}, no card over its limit")); - machine.cards[0].usable_bytes += over; - } - Err(PlaceError::PromptOver { - reserve: got, - left, - short, - .. - }) => { - assert_eq!((got, short), (reserve, reserve - left)); - break short; - } - other => panic!("{other:?}, not the reserve's refusal"), - } - }; - machine.cards[0].usable_bytes += short - 1; - assert!(matches!( - inputs.plan(&machine, 64, &levers), - Err(PlaceError::PromptOver { short: 1, .. }) - )); + let host_only = crate::placement::plan_host_routed( + &inputs.model, + &machine, + 64, + &inputs.seq_terms(), + &levers, + ) + .expect("the host-only reference plans"); + let dense = crate::arch::place_hybrid::card_terms(&host_only.cards[0]); + machine.cards[0].usable_bytes = machine.cards[0].margin_bytes + dense + reserve - 1; + match inputs.plan(&machine, 64, &levers) { + Err(PlaceError::PromptOver { + reserve: got, + left, + short, + .. + }) => assert_eq!((got, left, short), (reserve, reserve - 1, 1)), + other => panic!("{other:?}, not the reserve's one-byte refusal"), + } machine.cards[0].usable_bytes += 1; let plan = inputs .plan(&machine, 64, &levers) From a3206fe49a3e6cfb6c88e8160a9be51509a38875 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:55:40 +0900 Subject: [PATCH 12/19] Use one resident sequence for the Solar reference budget --- crates/model/src/arch/solar2/place.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/model/src/arch/solar2/place.rs b/crates/model/src/arch/solar2/place.rs index 4de9ecd33..6f0af8616 100644 --- a/crates/model/src/arch/solar2/place.rs +++ b/crates/model/src/arch/solar2/place.rs @@ -404,7 +404,7 @@ mod tests { &inputs.model, &machine, 64, - &inputs.seq_terms(), + &inputs.seq_terms().slots_of(1), &levers, ) .expect("the host-only reference plans"); From 6ecd144b3fda017388440e050258dfd2ffbc9b7f Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:57:52 +0900 Subject: [PATCH 13/19] Exclude reusable granule padding from the no-expert-room boundary --- crates/model/src/arch/solar2/place.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/model/src/arch/solar2/place.rs b/crates/model/src/arch/solar2/place.rs index 6f0af8616..d6051fae9 100644 --- a/crates/model/src/arch/solar2/place.rs +++ b/crates/model/src/arch/solar2/place.rs @@ -400,6 +400,9 @@ mod tests { ); let levers = PlanLevers::default(); let mut machine = workstation::plan_gate(LAYERS); + // Rounded dense padding may hold experts for no extra card bytes. + // Byte granules isolate the reserve's exact no-expert-room boundary. + machine.cards[0].granule_bytes = 1; let host_only = crate::placement::plan_host_routed( &inputs.model, &machine, From f3b2a21199126a46b87979cb677eb7447a9f27f5 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 08:02:27 +0900 Subject: [PATCH 14/19] Use the nonzero byte-granule constant in the reserve fixture --- crates/model/src/arch/solar2/place.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/model/src/arch/solar2/place.rs b/crates/model/src/arch/solar2/place.rs index d6051fae9..849e38fba 100644 --- a/crates/model/src/arch/solar2/place.rs +++ b/crates/model/src/arch/solar2/place.rs @@ -402,7 +402,7 @@ mod tests { let mut machine = workstation::plan_gate(LAYERS); // Rounded dense padding may hold experts for no extra card bytes. // Byte granules isolate the reserve's exact no-expert-room boundary. - machine.cards[0].granule_bytes = 1; + machine.cards[0].granule_bytes = std::num::NonZeroU64::MIN; let host_only = crate::placement::plan_host_routed( &inputs.model, &machine, From 7e37ef9a6ebbd766032ee521a0ee16e25f163851 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 08:04:45 +0900 Subject: [PATCH 15/19] Document the exact-granule reserve boundary --- docs/research/flint-attach.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 013052334..4bf20e2ab 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -113,7 +113,8 @@ each candidate. Fixed-reserve cards need no selected-form cache. The Solar budget regression starts from a typed mixed GGUF description and exercises the planner's one-byte refusal/success boundary without overwriting its described -prompt reserve. The exact dense-and-prompt budget leaves no room for expert weights. +prompt reserve. With byte granules (no reusable dense padding), the exact dense-and-prompt +budget leaves no room for expert weights. The real mixed-wrapper hardware case also checks allocation of a configured empty tier. Historical exact-source proof at `3071a9f0d8ed4773692e0c040dece222b90f6bec`: From cf035b8f68e96361a8eb56691a12b72be410032f Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:31:42 +0900 Subject: [PATCH 16/19] Document exact-source mixed-tier review consumers --- docs/research/flint-attach.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 4bf20e2ab..aa2c04739 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -126,6 +126,18 @@ Its fresh4bin/8carrier canonical comparison again found all1657entries identical with non-JIT rows equal. Native formatting/Clippy passed (warning3=ratchet3); model557 and placement202 passed,0 failed,5 model tests ignored,0 filtered. +The reviewed contract source `7e37ef9a6ebbd766032ee521a0ee16e25f163851` +executed the real CUDA mixed/empty-tier consumer on RTX3090 cc8.6:1 passed, +0 failed/ignored,427 filtered,23.98s. Its metadata-only +`a_dense_header_plans_through_qwen35_on` frontend consumer also passed1 case, +including the named dense-header expert-tier refusal; that is not a second GPU case. +The review code passed native formatting/all-target Clippy (warning3=ratchet3), +chain23, GGUF67, model557 and placement202 (five model cases ignored,0 filtered). +An exact-source7e37ef9a canonical comparison against main15778e2c found all1657 +entries identical across4bins/8carriers and non-JIT rows equal; it did not compare +driver-JIT resource rows. The older full-library counts above remain historical, +not a new full-library run of the review repairs. + The host-only public planner consumer covers one-byte boundaries before the first resident layer and before adding another down form, plus descriptor overflow refusal. These and the CUDA consumer are not a whole-model oracle or every affected model/residency gate; canonical equality does not From fd2a64b37f759063190233185bdaa1adfe846774 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:16:41 +0900 Subject: [PATCH 17/19] Document current-main dense-header consumer execution --- docs/research/flint-attach.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 0af54f40f..b05e17b4b 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -140,9 +140,16 @@ not a new full-library run of the review repairs. The component subsequently merges pinned upstream `cb8d92ebddd1ce1a325a15450c06a01f57ecf88c`. That reconciliation retains upstream's GLM mixer-form consumers alongside this component's -checked tier-byte API; the explicit-form API has no input-scale flag. The execution results -above belong to their named historical sources, not the merged source. No new runtime or -canonical-equivalence result is claimed by this merge. +checked tier-byte API; the explicit-form API has no input-scale flag. The GPU results +above belong to their named historical sources, not the merged source. + +At merged source `c38d603ca1d33c6c2bede63aba102199715fb9ef`, native formatting and +all-target Clippy passed, as did chain23, GGUF67, model559 and placement202 +(five model cases ignored, zero failed/filtered). The actual oxide-linked +`a_dense_header_plans_through_qwen35_on` frontend consumer passed1 case, +0 failed/ignored,6 filtered, including the named expert-tier refusal. +This is metadata-only host execution, not fresh NVIDIA execution of the merged source +or a whole-model numerical result. The host-only public planner consumer covers one-byte boundaries before the first resident layer and before adding another down form, plus descriptor overflow refusal. These and the CUDA consumer From 462213ca4afaaa7757644567eecd5976302a7833 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:53:07 +0900 Subject: [PATCH 18/19] Fix Qwen tier descriptor down-form bounds --- crates/gpu-gates/src/bin/shared/qwen3moe_place.rs | 4 ++-- crates/model/src/arch/qwen35moe/place.rs | 11 ++++++----- crates/model/src/arch/qwen3moe/place.rs | 11 ++++++----- crates/model/tests/qwen3_bp_plan.rs | 4 ++-- docs/research/flint-attach.md | 8 ++++++-- 5 files changed, 22 insertions(+), 16 deletions(-) diff --git a/crates/gpu-gates/src/bin/shared/qwen3moe_place.rs b/crates/gpu-gates/src/bin/shared/qwen3moe_place.rs index bb38199a7..74d7c093a 100644 --- a/crates/gpu-gates/src/bin/shared/qwen3moe_place.rs +++ b/crates/gpu-gates/src/bin/shared/qwen3moe_place.rs @@ -271,8 +271,8 @@ fn laid( [stage] => workstation::placed_with_arena(*stage, layers, arena), [stage, tier] => { let batch = match inputs { - Inputs::Qwen3(i) => q3::tier_batch(&i.hp), - Inputs::Qwen35(i) => q35::leg_tier_batch(&i.spec), + Inputs::Qwen3(i) => q3::tier_batch(i), + Inputs::Qwen35(i) => q35::leg_tier_batch(i), }?; let mut machine = workstation::plan_tiers(layers, *stage, &[*tier], None, batch)?; q3::stage_placed(&mut machine, arena); diff --git a/crates/model/src/arch/qwen35moe/place.rs b/crates/model/src/arch/qwen35moe/place.rs index 7433ccbc3..c85d9bbb0 100644 --- a/crates/model/src/arch/qwen35moe/place.rs +++ b/crates/model/src/arch/qwen35moe/place.rs @@ -52,7 +52,7 @@ use super::hparams::Kind; use super::{mtp, names}; use crate::arch::{architecture_is, coverage}; use crate::placement::accepts::{ - Program, leg_card_format, program_of, qwen38_down, qwen38_gate_up, routed_rule, + Program, leg_card_format, leg_down_forms, program_of, qwen38_down, qwen38_gate_up, routed_rule, }; use crate::placement::ctx::{CtxLimit, PastLimit, check_slot_ctx}; use crate::placement::ctx_ask::PlanRefusal; @@ -1498,18 +1498,19 @@ pub const LEG_TIER_CHUNK: u64 = 8; /// card's staging, one chunk of the leg's scratch and the host's rows, for /// blocks of the host union's columns. [`tier_batch`] is the qwen4exp plan's /// figure. -pub fn leg_tier_batch(spec: &ModelSpec) -> Result { +pub fn leg_tier_batch(inputs: &PlanInputs) -> Result { + let spec = &inputs.spec; let moe = spec.routed().ok_or_else(|| PlacementError::Metadata { key: "Qwen3.6 expert tier".into(), detail: "a dense model has no routed experts for an expert tier".into(), })?; let (expert_ff, n_used) = (u64::from(moe.expert_ff), u64::from(moe.top_k)); - workstation::tier_leg_batch_bytes( + workstation::try_tier_leg_batch_bytes_of( u64::from(spec.hidden), expert_ff, n_used, - LEG_TIER_CHUNK, - crate::moe::UNION_MAX_COLS as u64, + (LEG_TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64), + leg_down_forms(Program::Qwen35Body, &inputs.model), ) } diff --git a/crates/model/src/arch/qwen3moe/place.rs b/crates/model/src/arch/qwen3moe/place.rs index 8afc5c3ed..7d2aa664a 100644 --- a/crates/model/src/arch/qwen3moe/place.rs +++ b/crates/model/src/arch/qwen3moe/place.rs @@ -19,7 +19,7 @@ use gguf::Split; use super::hparams::Hparams; use super::roles; -use crate::placement::accepts::{Program, routed_rule}; +use crate::placement::accepts::{Program, leg_down_forms, routed_rule}; use crate::placement::ctx_ask::PlanRefusal; use crate::placement::workstation::{self, TierBatchBytes}; use crate::placement::{ @@ -45,13 +45,14 @@ pub const TIER_CHUNK: u64 = 8; /// The expert tier's prompt-batch bytes ([`TierBatchBytes`]) for blocks of /// the host union's columns ([`crate::moe::UNION_MAX_COLS`]): the tier card's /// staging, one chunk of the card leg's scratch and the host's rows. -pub fn tier_batch(hp: &Hparams) -> Result { - workstation::tier_leg_batch_bytes( +pub fn tier_batch(inputs: &PlanInputs) -> Result { + let hp = &inputs.hp; + workstation::try_tier_leg_batch_bytes_of( hp.n_embd as u64, hp.experts.ff as u64, hp.experts.n_used as u64, - TIER_CHUNK, - crate::moe::UNION_MAX_COLS as u64, + (TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64), + leg_down_forms(PROGRAM, &inputs.model), ) } diff --git a/crates/model/tests/qwen3_bp_plan.rs b/crates/model/tests/qwen3_bp_plan.rs index 64175fc36..da3cf6dc0 100644 --- a/crates/model/tests/qwen3_bp_plan.rs +++ b/crates/model/tests/qwen3_bp_plan.rs @@ -522,7 +522,7 @@ fn qwen36_inputs() -> q35::PlanInputs { fn qwen36_tier(stage: CardSpec, tier: CardSpec) -> Result<(u64, u64, u64), Place35Error> { let inputs = qwen36_inputs(); - let machine = tiered(Q36_LAYERS, stage, tier, q35::leg_tier_batch(&inputs.spec)?); + let machine = tiered(Q36_LAYERS, stage, tier, q35::leg_tier_batch(&inputs)?); let plan = inputs.plan_rule( &machine, CTX, @@ -541,7 +541,7 @@ fn qwen3_30b_q8_bp_tier_experts() { Q3_LAYERS, A6000, RTX_3090, - q3::tier_batch(&inputs.hp).expect("valid tier geometry"), + q3::tier_batch(&inputs).expect("valid tier geometry"), ); let plan = inputs .plan(&machine, CTX, &PlanLevers::default()) diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index b05e17b4b..c8697b9f6 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -99,8 +99,12 @@ budget regression exercises this distinction without a GPU, including OS reserve `Result` and construct candidate-selected descriptors. An empty tier selects no down form, matching `LegTier`'s actual allocation instead of advertising unused Q8Act scratch. Qwen3/Qwen35, MiMo and Muse use this checked path. -MiniMax obtains its forms from `accepts::leg_down_forms` under its selected program, -as GLM5Next and Solar do; no fallback format can admit a mismatched triple. +Qwen3/Qwen35 and MiniMax derive their descriptor bounds from the described model's +admitted forms through `accepts::leg_down_forms`, as GLM5Next and Solar do. +The Qwen public wrappers take `PlanInputs`, not geometry alone: a supported +Q8_0, Q5_1 or IQ4_NL candidate must not fail the whole plan just because its +32-value form remains unselected at the tier's budget boundary. +No fallback format can admit a mismatched triple. The checked constructors and non-leg paths share staging and host byte formulas. Malformed geometry and overflow are refused rather than panicking; a dense Qwen3.6 request for an expert tier is a named metadata refusal, not zero expert geometry. From 800700db14acd5d0ccf825547f06b40ca62be091 Mon Sep 17 00:00:00 2001 From: midagedev Date: Tue, 27 Oct 2026 16:08:28 +0900 Subject: [PATCH 19/19] placement: a tier card with no leg descriptor keeps its reserve; the overflow test names its panic Co-Authored-By: Claude Opus 5.5 --- crates/placement/src/kernels.rs | 2 +- crates/placement/src/placement.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/placement/src/kernels.rs b/crates/placement/src/kernels.rs index 8a8443f86..5aefc70ec 100644 --- a/crates/placement/src/kernels.rs +++ b/crates/placement/src/kernels.rs @@ -451,7 +451,7 @@ mod tests { } #[test] - #[should_panic] + #[should_panic(expected = "leg_bytes: scratch bytes overflow u64")] fn infallible_leg_sizing_refuses_overflow() { let _ = leg_bytes(1 << 61, 8, 256, 256, DownForms::of(DownAct::Q8Act)); } diff --git a/crates/placement/src/placement.rs b/crates/placement/src/placement.rs index 2d57f3239..f470d2617 100644 --- a/crates/placement/src/placement.rs +++ b/crates/placement/src/placement.rs @@ -4473,7 +4473,7 @@ impl Plan<'_> { if rows.next().is_some() { return Ok(None); } - if name != workstation::TIER_BATCH_RESERVE { + if name != workstation::TIER_BATCH_RESERVE || card.tier_leg.is_none() { return Ok(Some(*bytes)); } let counts = match c.checked_sub(self.machine.cards.len()) {