From aa574d0b987c0d027aff0a8efa48f8496c4c9a94 Mon Sep 17 00:00:00 2001
From: BHCC2025 <241330250+BHCC2025@users.noreply.github.com>
Date: Tue, 27 Oct 2026 09:06:10 -0500
Subject: [PATCH 1/8] exl3 cuda: per-weight plans, mid-M kernels and linear_wc
 for 17-128 rows, projection groups, PDL (same bits)

The Python engine's EXL3 speed series (#260) on the native Dense module. Weight.sk/wk let a family fix a shape's
plan (K splits, warps); the mid-M kernels and linear_wc give the linear's bits at 17-128 rows; matmulGroup runs up to
three layers of one K over the same rows in one rotation and one linear launch; the rotations and decode-row linears
launch with programmatic dependent launch (TF_EXL3_PDL=0: plain), and counters are zeroed once.
tf-cuda-test exl3-plans, exl3-midm and exl3-shapes check and time them.
---
 zig/kernels/cuda/exl3_linear.cu     | 217 +++++++++++++++++++++-
 zig/kernels/cuda/exl3_linear_wc.cuh | 223 +++++++++++++++++++++++
 zig/src/cuda/exl3.zig               | 186 ++++++++++++++++++-
 zig/tests/cuda/exl3_plan_bench.zig  | 271 ++++++++++++++++++++++++++++
 zig/tests/cuda/main.zig             |   7 +
 5 files changed, 887 insertions(+), 17 deletions(-)
 create mode 100644 zig/kernels/cuda/exl3_linear_wc.cuh
 create mode 100644 zig/tests/cuda/exl3_plan_bench.zig

diff --git a/zig/kernels/cuda/exl3_linear.cu b/zig/kernels/cuda/exl3_linear.cu
index 140e16d570..42535726a4 100644
--- a/zig/kernels/cuda/exl3_linear.cu
+++ b/zig/kernels/cuda/exl3_linear.cu
@@ -100,26 +100,49 @@ __device__ __forceinline__ void step_lane_words(const uint32_t (&raw)[8 * lane_w
     for (int q = 0; q < lane_words<K2>(); ++q) w[q] = raw[j * lane_words<K2>() + q];
 }
 
+// Programmatic dependent launch: wait for the previous kernel's writes here (a no-op without it); trigger the next.
+__device__ __forceinline__ void pdl_wait() { asm volatile("griddepcontrol.wait;\n" ::: "memory"); }
+__device__ __forceinline__ void pdl_trigger() { asm volatile("griddepcontrol.launch_dependents;\n" ::: "memory"); }
+
 // xh = fp16(((x * suh) @ H) / sqrt(128)): one warp per 128 columns, one block of 4 warps per 512.
-extern "C" __global__ void __launch_bounds__(128) tf_exl3_rot_in(RotInArgs a) {
+__device__ __forceinline__ void rot_in_body(const void* x, int x_dtype, const half* suh, half* xh, int kdim) {
+    pdl_trigger();
+    pdl_wait();
     const int blk = blockIdx.x * 4 + (threadIdx.x >> 5), row = blockIdx.y, lane = threadIdx.x & 31;
-    if (blk * 128 >= a.k) return;
+    if (blk * 128 >= kdim) return;
     const int k = blk * 128 + 4 * lane;
     float v[4], s[4];
-    load4(a.x, a.x_dtype, (size_t)row * a.k + k, v);
-    load4(a.suh, F16, k, s);
+    load4(x, x_dtype, (size_t)row * kdim + k, v);
+    load4(suh, F16, k, s);
 #pragma unroll
     for (int j = 0; j < 4; ++j) v[j] *= s[j];
     fwht128(v, lane);
 #pragma unroll
     for (int j = 0; j < 4; ++j) v[j] *= HAD_SCALE;
-    store4(a.xh, F16, (size_t)row * a.k + k, v);
+    store4(xh, F16, (size_t)row * kdim + k, v);
+}
+
+extern "C" __global__ void __launch_bounds__(128) tf_exl3_rot_in(RotInArgs a) {
+    rot_in_body(a.x, a.x_dtype, a.suh, a.xh, a.k);
+}
+
+// Up to three layers that read the same rows (a projection group): member blockIdx.z's rotation, one launch.
+struct RotGroupArgs {
+    const void* x;
+    const half* suh[3];
+    half* xh[3];
+    int x_dtype;
+    int m, k;
+};
+
+extern "C" __global__ void __launch_bounds__(128) tf_exl3_rot_group(RotGroupArgs a) {
+    rot_in_body(a.x, a.x_dtype, a.suh[blockIdx.z], a.xh[blockIdx.z], a.k);
 }
 
 
 // One program: (N / 128) * SK blocks, split-major like a (N / 128, SK) grid, WK warps each.
 template <int K2, int CB, int WK>
-__device__ __forceinline__ void linear_body(LinearArgs a) {
+__device__ __forceinline__ void linear_body(const LinearArgs& a, int bid) {
     constexpr int TW = tile_words<K2>(), LW = lane_words<K2>();
     constexpr bool PF = K2 <= 12;                             // the next k step's words load while this one decodes
     constexpr int SR = 8 * LW;
@@ -127,7 +150,7 @@ __device__ __forceinline__ void linear_body(LinearArgs a) {
     __shared__ int last;
     const int M = a.m, K = a.k, N = a.n, SK = a.sk, NB = N >> 7;
     const int RH = imin(M, 8);                                // rows of red a warp
-    const int nb = (int)blockIdx.x % NB, split = (int)blockIdx.x / NB;
+    const int nb = bid % NB, split = bid / NB;
     const int warp = threadIdx.x >> 5, lane = threadIdx.x & 31;
     const int g = lane >> 2, t = lane & 3;
     const int per_warp = (K >> 4) / SK / WK;
@@ -135,6 +158,9 @@ __device__ __forceinline__ void linear_body(LinearArgs a) {
     const uint32_t* tiles = a.t + nb * a.stride_nb;
     const int col0 = nb * 128;
 
+    // (prefetching the first k steps to L2 before the wait measured a millisecond slower a token on GB10)
+    pdl_wait();
+
     for (int m0 = 0, pass = 0; m0 < M; m0 += 16, ++pass) {
         const int R = imin(16, M - m0);
 
@@ -184,6 +210,7 @@ __device__ __forceinline__ void linear_body(LinearArgs a) {
             }
         }
 
+        if (m0 == 0) pdl_trigger();                  // the next kernel may launch: only the reduction is left
         // the warps' sums, added in warp order, rows 0-7 of the pass and then rows 8-15
         for (int rlo = 0; rlo < R; rlo += 8) {
             const int rn = imin(R - rlo, 8);
@@ -252,15 +279,163 @@ __device__ __forceinline__ void linear_body(LinearArgs a) {
     }
 }
 
+// 17-128 rows (mid-M): a program decodes each k step's tiles once for P 16-row passes, in linear_body's order and bits.
+template <int K2, int CB, int WK, int P>
+__device__ __forceinline__ void linear_mpg_body(LinearArgs a) {
+    constexpr int TW = tile_words<K2>(), LW = lane_words<K2>();
+    constexpr bool PF = K2 <= 12;
+    constexpr int SR = 8 * LW;
+    extern __shared__ __align__(16) float red[];              // WK * RH * 128 floats
+    __shared__ int last;
+    const int M = a.m, K = a.k, N = a.n, SK = a.sk, NB = N >> 7;
+    const int RH = imin(M, 8);
+    const int G = (M + 16 * P - 1) / (16 * P), grp = (int)blockIdx.x % G, nb = (int)blockIdx.x / G, split = blockIdx.y;
+    const int warp = threadIdx.x >> 5, lane = threadIdx.x & 31;
+    const int g = lane >> 2, t = lane & 3;
+    const int per_warp = (K >> 4) / SK / WK;
+    const int kt0 = split * (per_warp * WK) + warp * per_warp;
+    const uint32_t* tiles = a.t + nb * a.stride_nb;
+    const int col0 = nb * 128;
+    const int mg = grp * 16 * P, pg = grp * P;
+
+    float acc[P][8][2][4];
+#pragma unroll
+    for (int q = 0; q < P; ++q)
+#pragma unroll
+        for (int i = 0; i < 8; ++i)
+#pragma unroll
+            for (int h = 0; h < 2; ++h)
+#pragma unroll
+                for (int c = 0; c < 4; ++c) acc[q][i][h][c] = 0.f;
+
+    // each pass's rows for the mma, clamped inside the buffer (rows past M are computed and dropped)
+    const half* x0[P];
+    const half* x1[P];
+#pragma unroll
+    for (int q = 0; q < P; ++q) {
+        const int mq = imin(mg + 16 * q, M - 1), Rq = imin(16, M - mq) < 1 ? 1 : imin(16, M - mq);
+        const int r0 = mq + (g < Rq ? g : Rq - 1), r1 = mq + (g + 8 < Rq ? g + 8 : Rq - 1);
+        x0[q] = static_cast<const half*>(a.xh) + (size_t)r0 * K;
+        x1[q] = static_cast<const half*>(a.xh) + (size_t)r1 * K;
+    }
+    const uint32_t* tile = tiles + (size_t)kt0 * a.stride_k;
+    uint32_t cur[PF ? SR : 1], nxt[PF ? SR : 1];
+    if constexpr (PF) load_step<K2>(tile, lane, cur);
+#pragma unroll 1
+    for (int i = 0; i < per_warp; ++i) {
+        const int kt = kt0 + i;
+        uint32_t u[P][4];
+#pragma unroll
+        for (int q = 0; q < P; ++q) {
+            u[q][0] = __ldg(reinterpret_cast<const uint32_t*>(x0[q] + kt * 16 + 2 * t));
+            u[q][2] = __ldg(reinterpret_cast<const uint32_t*>(x0[q] + kt * 16 + 2 * t + 8));
+            u[q][1] = __ldg(reinterpret_cast<const uint32_t*>(x1[q] + kt * 16 + 2 * t));
+            u[q][3] = __ldg(reinterpret_cast<const uint32_t*>(x1[q] + kt * 16 + 2 * t + 8));
+        }
+        if constexpr (PF) {
+            if (i + 1 < per_warp) load_step<K2>(tile + (size_t)(i + 1) * a.stride_k, lane, nxt);
+        }
+#pragma unroll
+        for (int j = 0; j < 8; ++j) {
+            uint32_t w[LW];
+            if constexpr (PF) step_lane_words<K2>(cur, j, w);
+            else ldg_lane_words<K2>(tile + (size_t)i * a.stride_k + j * TW, lane, w);
+            uint32_t b0[2], b1[2];
+            decode_lane<K2, CB>(w, lane, b0, b1);
+#pragma unroll
+            for (int q = 0; q < P; ++q) {
+                mma16816(acc[q][j][0], u[q], b0);
+                mma16816(acc[q][j][1], u[q], b1);
+            }
+        }
+        if constexpr (PF) {
+#pragma unroll
+            for (int q = 0; q < SR; ++q) cur[q] = nxt[q];
+        }
+    }
+
+#pragma unroll
+    for (int q = 0; q < P; ++q) {
+        const int m0 = mg + 16 * q, pass = pg + q;
+        if (m0 >= M) break;
+        const int R = imin(16, M - m0);
+        // the warps' sums, added in warp order, rows 0-7 of the pass and then rows 8-15
+        for (int rlo = 0; rlo < R; rlo += 8) {
+            const int rn = imin(R - rlo, 8);
+            __syncthreads();
+            if (g < RH) {
+#pragma unroll
+                for (int i = 0; i < 8; ++i)
+#pragma unroll
+                    for (int h = 0; h < 2; ++h) {
+                        const int col = i * 16 + h * 8 + 2 * t;
+                        *reinterpret_cast<float2*>(red + (warp * RH + g) * 128 + col) =
+                            rlo ? make_float2(acc[q][i][h][2], acc[q][i][h][3])
+                                : make_float2(acc[q][i][h][0], acc[q][i][h][1]);
+                    }
+            }
+            __syncthreads();
+            if (SK == 1) {
+                for (int r = warp; r < rn; r += WK) {
+                    float v[4];
+                    const float4 s4 = *reinterpret_cast<const float4*>(red + r * 128 + 4 * lane);
+                    v[0] = s4.x; v[1] = s4.y; v[2] = s4.z; v[3] = s4.w;
+#pragma unroll
+                    for (int w = 1; w < WK; ++w) {
+                        const float4 o = *reinterpret_cast<const float4*>(red + (w * RH + r) * 128 + 4 * lane);
+                        v[0] += o.x; v[1] += o.y; v[2] += o.z; v[3] += o.w;
+                    }
+                    finish(v, lane, a.svh, a.bias, col0 + 4 * lane);
+                    store4(a.y, a.y_dtype, (size_t)(m0 + rlo + r) * N + col0 + 4 * lane, v);
+                }
+            } else {
+                for (int idx = threadIdx.x; idx < rn * 32; idx += WK * 32) {
+                    const int r = idx >> 5, c = 4 * (idx & 31);
+                    float4 s4 = *reinterpret_cast<const float4*>(red + r * 128 + c);
+#pragma unroll
+                    for (int w = 1; w < WK; ++w) {
+                        const float4 o = *reinterpret_cast<const float4*>(red + (w * RH + r) * 128 + c);
+                        s4.x += o.x; s4.y += o.y; s4.z += o.z; s4.w += o.w;
+                    }
+                    *reinterpret_cast<float4*>(a.z + ((size_t)split * M + m0 + rlo + r) * N + col0 + c) = s4;
+                }
+            }
+        }
+        if (SK > 1) {
+            __threadfence();
+            __syncthreads();
+            if (threadIdx.x == 0) last = atomicAdd(a.counters + pass * NB + nb, 1) == SK - 1;
+            __syncthreads();
+            if (last) {
+                __threadfence();
+                for (int r = warp; r < R; r += WK) {
+                    const size_t at = ((size_t)m0 + r) * N + col0 + 4 * lane;
+                    float4 s4 = __ldcg(reinterpret_cast<const float4*>(a.z + at));
+                    for (int o = 1; o < SK; ++o) {
+                        const float4 v4 = __ldcg(reinterpret_cast<const float4*>(a.z + (size_t)o * M * N + at));
+                        s4.x += v4.x; s4.y += v4.y; s4.z += v4.z; s4.w += v4.w;
+                    }
+                    float v[4] = {s4.x, s4.y, s4.z, s4.w};
+                    finish(v, lane, a.svh, a.bias, col0 + 4 * lane);
+                    store4(a.y, a.y_dtype, (size_t)(m0 + r) * N + col0 + 4 * lane, v);
+                }
+                if (threadIdx.x == 0) a.counters[pass * NB + nb] = 0;
+            }
+        }
+    }
+}
+
+#include "exl3_linear_wc.cuh"
+
 // One extern "C" kernel per instantiation: the fatbin carries __global__ functions, the host resolves them by name.
 #define TF_EXL3_LINEAR(K2, CB, WK)                                                       \
     extern "C" __global__ void __launch_bounds__((WK) * 32)                              \
         tf_exl3_linear_##K2##_##CB##_##WK(tf_exl3::LinearArgs a) {                       \
-        tf_exl3::linear_body<K2, CB, WK>(a);                                             \
+        tf_exl3::linear_body<K2, CB, WK>(a, blockIdx.x);                                 \
     }
 
 // Every width the format serves (linear.cu TF_EXL3_ALL plus 4.5-6.5): whole bits, all codebooks; 7.5 is not a width.
-#define TF_EXL3_WIDTHS(K2, CB) TF_EXL3_LINEAR(K2, CB, 4) TF_EXL3_LINEAR(K2, CB, 8)
+#define TF_EXL3_WIDTHS(K2, CB) TF_EXL3_LINEAR(K2, CB, 2) TF_EXL3_LINEAR(K2, CB, 4) TF_EXL3_LINEAR(K2, CB, 8)
 #define TF_EXL3_CODEBOOK(CB)                                                                                \
     TF_EXL3_WIDTHS(2, CB) TF_EXL3_WIDTHS(4, CB) TF_EXL3_WIDTHS(6, CB) TF_EXL3_WIDTHS(8, CB)                 \
     TF_EXL3_WIDTHS(10, CB) TF_EXL3_WIDTHS(12, CB) TF_EXL3_WIDTHS(14, CB) TF_EXL3_WIDTHS(16, CB)
@@ -269,6 +444,30 @@ TF_EXL3_WIDTHS(3, 2) TF_EXL3_WIDTHS(5, 2) TF_EXL3_WIDTHS(7, 2) TF_EXL3_WIDTHS(9,
 TF_EXL3_WIDTHS(11, 2) TF_EXL3_WIDTHS(13, 2)
 TF_EXL3_CODEBOOK(1) // mcg
 TF_EXL3_CODEBOOK(0) // inst3
+// The mid-M kernels for the mul1 widths a dense family serves (3 to 6 bits), P = 1 (17-48 rows) and 2 (49-128).
+#define TF_EXL3_MPG(K2, WK, P)                                                           \
+    extern "C" __global__ void __launch_bounds__((WK) * 32)                              \
+        tf_exl3_mpg_##K2##_##WK##_##P(tf_exl3::LinearArgs a) {                           \
+        tf_exl3::linear_mpg_body<K2, 2, WK, P>(a);                                       \
+    }
+#define TF_EXL3_MPG_W(K2) TF_EXL3_MPG(K2, 2, 1) TF_EXL3_MPG(K2, 4, 1) TF_EXL3_MPG(K2, 8, 1) \
+    TF_EXL3_MPG(K2, 2, 2) TF_EXL3_MPG(K2, 4, 2) TF_EXL3_MPG(K2, 8, 2)
+TF_EXL3_MPG_W(6) TF_EXL3_MPG_W(8) TF_EXL3_MPG_W(10) TF_EXL3_MPG_W(12)
+#undef TF_EXL3_MPG_W
+#undef TF_EXL3_MPG
+// A projection group: up to three layers of one K and warps, one launch; member i takes blocks ends[i - 1]..ends[i].
+struct GroupArgs {
+    LinearArgs l[3];
+    int ends[3];
+};
+#define TF_EXL3_GROUP(K2, CB, WK)                                                        \
+    extern "C" __global__ void __launch_bounds__((WK) * 32)                              \
+        tf_exl3_group_##K2##_##CB##_##WK(tf_exl3::GroupArgs g) {                         \
+        const int b = blockIdx.x, i = b < g.ends[0] ? 0 : b < g.ends[1] ? 1 : 2;         \
+        tf_exl3::linear_body<K2, CB, WK>(g.l[i], b - (i ? g.ends[i - 1] : 0));           \
+    }
+TF_EXL3_GROUP(8, 2, 4) TF_EXL3_GROUP(8, 2, 8)
+#undef TF_EXL3_GROUP
 #undef TF_EXL3_CODEBOOK
 #undef TF_EXL3_WIDTHS
 }  // namespace tf_exl3
diff --git a/zig/kernels/cuda/exl3_linear_wc.cuh b/zig/kernels/cuda/exl3_linear_wc.cuh
new file mode 100644
index 0000000000..2737a6667d
--- /dev/null
+++ b/zig/kernels/cuda/exl3_linear_wc.cuh
@@ -0,0 +1,223 @@
+// EXL3 linear for 17-128 rows of 4- and 6-bit mul1 layers, a 128-column block a program, in linear_body's bits.
+
+// LinearArgs, then the plan's warps (the K ranges a split holds), packed as the host adds them.
+struct WcArgs {
+    const void* xh;
+    const uint32_t* t;
+    long long stride_k;
+    long long stride_nb;
+    const half* svh;
+    const half* bias;
+    void* y;
+    float* z;
+    int* counters;
+    int y_dtype;
+    int n, sk, m, k;
+    int wk;
+};
+
+__device__ __forceinline__ void wc_cp16(void* dst, const void* src) {
+    asm volatile("cp.async.cg.shared.global [%0], [%1], 16;\n" ::"r"((uint32_t)__cvta_generic_to_shared(dst)), "l"(src));
+}
+__device__ __forceinline__ void wc_commit() { asm volatile("cp.async.commit_group;\n" ::); }
+template <int N_>
+__device__ __forceinline__ void wc_wait() { asm volatile("cp.async.wait_group %0;\n" ::"n"(N_)); }
+__device__ __forceinline__ void wc_ldsm4(uint32_t (&a)[4], const void* p) {
+    asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
+                 : "=r"(a[0]), "=r"(a[1]), "=r"(a[2]), "=r"(a[3])
+                 : "r"((uint32_t)__cvta_generic_to_shared(p)));
+}
+
+// A lane's words of one tile from shared memory (ldg_lane_words without the read-only cache).
+template <int K2>
+__device__ __forceinline__ void sh_lane_words(const uint32_t* tile, int lane, uint32_t (&w)[lane_words<K2>()]) {
+    int word, offset;
+    lane_start<K2>(lane, word, offset);
+#pragma unroll
+    for (int i = 0; i < lane_words<K2>(); ++i) w[i] = tile[(word + i) % tile_words<K2>()];
+}
+
+template <int K2, int RG, int KS, int NS>
+struct WcLayout {
+    static constexpr int RAW = 8 * tile_words<K2>();          // words of one k step's 8 tiles
+    static constexpr int RAW_STAGE = KS * RAW * 4;
+    static constexpr int A_STAGE = KS * RG * 32;               // the program's rows, 16 fp16 each a k step
+    static constexpr int STAGE = RAW_STAGE + A_STAGE;
+    static constexpr int PIPE = NS * STAGE;
+    static constexpr int RED = RG * 128 * 4;                   // the finished sums (reuses the ring)
+    static constexpr int SMEM = PIPE > RED ? PIPE : RED;
+};
+
+template <int K2, int MT, int KS, int NS, bool FOLD>
+__device__ __forceinline__ void linear_wc_body(const WcArgs& a) {
+    constexpr int CB = 2;                                      // mul1
+    constexpr int NW = 8, NTH = NW * 32, RG = 16 * MT;
+    constexpr int TW = tile_words<K2>(), LW = lane_words<K2>();
+    using Lay = WcLayout<K2, RG, KS, NS>;
+    extern __shared__ __align__(128) unsigned char smb[];
+    __shared__ int last;
+    const int M = a.m, K = a.k, N = a.n, SK = a.sk, WK = a.wk;
+    const int G = (M + RG - 1) / RG, grp = blockIdx.x % G, nb = blockIdx.x / G, split = blockIdx.y;
+    const int NB = N >> 7;
+    const int warp = threadIdx.x >> 5, lane = threadIdx.x & 31, g = lane >> 2, t = lane & 3;
+    const int per_warp = (K >> 4) / SK / WK, nks = per_warp * WK * (FOLD ? SK : 1), kt0 = FOLD ? 0 : split * nks;
+    const uint32_t* tiles = a.t + nb * a.stride_nb;
+    const half* xh = static_cast<const half*>(a.xh);
+    const int col0 = nb * 128, mg = grp * RG, rows = imin(RG, M - mg);
+    const int S = nks / KS;
+
+    // stage s: k steps s KS .. s KS + KS - 1 of the split into ring slot s % NS; always one commit group
+    auto issue = [&](int s) {
+        if (s < S) {
+            unsigned char* st = smb + (s % NS) * Lay::STAGE;
+            constexpr int RC = Lay::RAW / 4;
+            for (int c = threadIdx.x; c < KS * RC; c += NTH) {
+                const int ks = c / RC, off = c % RC;
+                wc_cp16(st + (ks * Lay::RAW + off * 4) * 4, tiles + (size_t)(kt0 + s * KS + ks) * a.stride_k + off * 4);
+            }
+            unsigned char* sa = st + Lay::RAW_STAGE;
+            for (int c = threadIdx.x; c < KS * RG * 2; c += NTH) {
+                const int ks = c / (RG * 2), r = (c >> 1) % RG, ch = c & 1;
+                const int row = imin(mg + r, M - 1);
+                wc_cp16(sa + (ks * RG + r) * 32 + ((ch ^ ((r >> 2) & 1)) << 4),
+                        xh + (size_t)row * K + (kt0 + s * KS + ks) * 16 + ch * 8);
+            }
+        }
+        wc_commit();
+    };
+
+    float acc[MT][2][4], tot[MT][2][4], gtot[FOLD ? MT : 1][2][4];
+#pragma unroll
+    for (int q = 0; q < MT; ++q)
+#pragma unroll
+        for (int h = 0; h < 2; ++h)
+#pragma unroll
+            for (int c = 0; c < 4; ++c) acc[q][h][c] = tot[q][h][c] = 0.f;
+
+#pragma unroll
+    for (int s = 0; s < NS - 1; ++s) issue(s);
+    const int lrow = (lane & 7) + 8 * ((lane >> 3) & 1), lch = lane >> 4;   // this lane's ldmatrix row and half
+    int r = 0, left = per_warp, sp = 0;                        // the current K range, its k steps to run; split
+#pragma unroll 1
+    for (int s = 0; s < S; ++s) {
+        wc_wait<NS - 2>();                                     // stage s has landed (this thread's copies) ...
+        __syncthreads();                                       // ... everyone's; slot s - 1 is free
+        issue(s + NS - 1);
+        const unsigned char* st = smb + (s % NS) * Lay::STAGE;
+        const uint32_t* raw = reinterpret_cast<const uint32_t*>(st);
+        const unsigned char* sa = st + Lay::RAW_STAGE;
+        uint32_t b[KS][2][2];
+#pragma unroll
+        for (int ks = 0; ks < KS; ++ks) {
+            uint32_t w[LW];
+            sh_lane_words<K2>(raw + ks * Lay::RAW + warp * TW, lane, w);
+            decode_lane<K2, CB>(w, lane, b[ks][0], b[ks][1]);
+        }
+#pragma unroll
+        for (int ks = 0; ks < KS; ++ks)
+#pragma unroll
+            for (int q = 0; q < MT; ++q) {
+                uint32_t af[4];
+                const int rr = 16 * q + lrow;
+                wc_ldsm4(af, sa + (ks * RG + rr) * 32 + ((lch ^ ((rr >> 2) & 1)) << 4));
+                mma16816(acc[q][0], af, b[ks][0]);
+                mma16816(acc[q][1], af, b[ks][1]);
+            }
+        left -= KS;
+        if (left == 0) {                                       // range r done: tot = s0, then tot += s_r
+#pragma unroll
+            for (int q = 0; q < MT; ++q)
+#pragma unroll
+                for (int h = 0; h < 2; ++h)
+#pragma unroll
+                    for (int c = 0; c < 4; ++c) {
+                        tot[q][h][c] = r ? tot[q][h][c] + acc[q][h][c] : acc[q][h][c];
+                        acc[q][h][c] = 0.f;
+                    }
+            ++r;
+            left = per_warp;
+            if constexpr (FOLD) {                              // every split in this program, summed in split order
+                if (r == WK) {
+#pragma unroll
+                    for (int q = 0; q < MT; ++q)
+#pragma unroll
+                        for (int h = 0; h < 2; ++h)
+#pragma unroll
+                            for (int c = 0; c < 4; ++c) gtot[q][h][c] = sp ? gtot[q][h][c] + tot[q][h][c] : tot[q][h][c];
+                    r = 0;
+                    ++sp;
+                }
+            }
+        }
+    }
+    if constexpr (FOLD) {
+#pragma unroll
+        for (int q = 0; q < MT; ++q)
+#pragma unroll
+            for (int h = 0; h < 2; ++h)
+#pragma unroll
+                for (int c = 0; c < 4; ++c) tot[q][h][c] = gtot[q][h][c];
+    }
+    wc_wait<0>();
+    __syncthreads();
+
+    float* red = reinterpret_cast<float*>(smb);
+#pragma unroll
+    for (int q = 0; q < MT; ++q) {
+        const int rr = 16 * q + g;
+#pragma unroll
+        for (int h = 0; h < 2; ++h) {
+            const int col = warp * 16 + h * 8 + 2 * t;
+            *reinterpret_cast<float2*>(red + rr * 128 + col) = make_float2(tot[q][h][0], tot[q][h][1]);
+            *reinterpret_cast<float2*>(red + (rr + 8) * 128 + col) = make_float2(tot[q][h][2], tot[q][h][3]);
+        }
+    }
+    __syncthreads();
+    if (FOLD || SK == 1) {
+        for (int rr = warp; rr < rows; rr += NW) {
+            const float4 u = *reinterpret_cast<const float4*>(red + rr * 128 + 4 * lane);
+            float v[4] = {u.x, u.y, u.z, u.w};
+            finish(v, lane, a.svh, a.bias, col0 + 4 * lane);
+            store4(a.y, a.y_dtype, (size_t)(mg + rr) * N + col0 + 4 * lane, v);
+        }
+        return;
+    }
+    for (int idx = threadIdx.x; idx < rows * 32; idx += NTH) {
+        const int rr = idx >> 5, c = 4 * (idx & 31);
+        *reinterpret_cast<float4*>(a.z + ((size_t)split * M + mg + rr) * N + col0 + c) =
+            *reinterpret_cast<const float4*>(red + rr * 128 + c);
+    }
+    __threadfence();
+    __syncthreads();
+    if (threadIdx.x == 0) last = atomicAdd(a.counters + grp * NB + nb, 1) == SK - 1;
+    __syncthreads();
+    if (last) {
+        __threadfence();
+        for (int rr = warp; rr < rows; rr += NW) {
+            const size_t at = ((size_t)mg + rr) * N + col0 + 4 * lane;
+            float4 sum = __ldcg(reinterpret_cast<const float4*>(a.z + at));
+            for (int q = 1; q < SK; ++q) {
+                const float4 w = __ldcg(reinterpret_cast<const float4*>(a.z + (size_t)q * M * N + at));
+                sum.x += w.x; sum.y += w.y; sum.z += w.z; sum.w += w.w;
+            }
+            float v[4] = {sum.x, sum.y, sum.z, sum.w};
+            finish(v, lane, a.svh, a.bias, col0 + 4 * lane);
+            store4(a.y, a.y_dtype, (size_t)(mg + rr) * N + col0 + 4 * lane, v);
+        }
+        if (threadIdx.x == 0) a.counters[grp * NB + nb] = 0;
+    }
+}
+
+// The host picks one by row count and shape (exl3.zig wcPick); grid (NB * ceil(M / RG), FOLD ? 1 : SK), 256 threads.
+#define TF_EXL3_WC(K2, MT, KS, F, FN)                                                    \
+    extern "C" __global__ void __launch_bounds__(256)                                    \
+        tf_exl3_wc_##K2##_##MT##_##KS##_##FN(tf_exl3::WcArgs a) {                        \
+        tf_exl3::linear_wc_body<K2, MT, KS, 4, F>(a);                                    \
+    }
+#define TF_EXL3_WC_K2(K2)                                                                                  \
+    TF_EXL3_WC(K2, 2, 2, false, 0) TF_EXL3_WC(K2, 2, 4, false, 0) TF_EXL3_WC(K2, 4, 2, false, 0)            \
+    TF_EXL3_WC(K2, 4, 4, false, 0) TF_EXL3_WC(K2, 2, 2, true, 1) TF_EXL3_WC(K2, 2, 4, true, 1)              \
+    TF_EXL3_WC(K2, 4, 2, true, 1) TF_EXL3_WC(K2, 4, 4, true, 1)
+TF_EXL3_WC_K2(8) TF_EXL3_WC_K2(12)
+#undef TF_EXL3_WC_K2
+#undef TF_EXL3_WC
diff --git a/zig/src/cuda/exl3.zig b/zig/src/cuda/exl3.zig
index da5c535dce..3920548ae0 100644
--- a/zig/src/cuda/exl3.zig
+++ b/zig/src/cuda/exl3.zig
@@ -1,6 +1,7 @@
 //! The EXL3 decode-row linear (exl3/linear.cu's device code): the plan, a layer's tensors, and the two kernels.
 const std = @import("std");
 const candidate = @import("exl3_candidate.zig");
+const platform = @import("sys");
 const driver = @import("driver.zig");
 const module = @import("module.zig");
 const launch_ = @import("launch.zig");
@@ -21,12 +22,12 @@ pub const Codebook = enum(u3) { inst3 = 0, mcg = 1, mul1 = 2 };
 /// One compiled kernel: 2 * bits halves, a codebook, and the warps one block runs.
 pub const Variant = struct { halves: u32, codebook: Codebook, warps: u32 };
 
-/// Every instantiation of exl3_linear.cu: every width (whole bits all codebooks, the mul1 half steps), 4 and 8 warps.
+/// Every instantiation of exl3_linear.cu: every width (whole bits, the mul1 half steps), 2, 4 and 8 warps.
 pub const variants = blk: {
-    var vs: [60]Variant = undefined;
+    var vs: [90]Variant = undefined;
     var i: usize = 0;
-    for (0..2) |w| {
-        const warps: u32 = if (w == 0) 4 else 8;
+    const counts = [_]u32{ 4, 8, 2 }; // 2 last: the 4- and 8-warp indices stay as they were
+    for (counts) |warps| {
         for ([_]u32{ 2, 1, 0 }) |cbi| {
             const cb: Codebook = @fromBackingInt(@intCast(cbi));
             var halves: u32 = 2;
@@ -52,6 +53,57 @@ pub fn symbol(comptime halves: u32, comptime codebook: Codebook, comptime warps:
 
 pub const rot_in_symbol = "tf_exl3_rot_in";
 
+/// The mid-M kernels (17-128 rows, the same bits as the linear's): mul1 at 3 to 6 bits, 2, 4 and 8 warps, P 1 and 2.
+pub const Mpg = struct { halves: u32, warps: u32, p: u32 };
+pub const mpg_variants = blk: {
+    var vs: [24]Mpg = undefined;
+    var i: usize = 0;
+    for ([_]u32{ 6, 8, 10, 12 }) |h| for ([_]u32{ 1, 2 }) |p| for ([_]u32{ 2, 4, 8 }) |w| {
+        vs[i] = .{ .halves = h, .warps = w, .p = p };
+        i += 1;
+    };
+    break :blk vs;
+};
+
+/// linear_wc (17-128 rows, 4- and 6-bit mul1): 16 mt rows a program, ks k steps a stage, fold: every split in one.
+pub const Wc = struct { halves: u32, mt: u32, ks: u32, fold: bool };
+pub const wc_variants = blk: {
+    var vs: [16]Wc = undefined;
+    var i: usize = 0;
+    for ([_]u32{ 8, 12 }) |h| for ([_]bool{ false, true }) |f| for ([_]u32{ 2, 4 }) |mt| for ([_]u32{ 2, 4 }) |ks| {
+        vs[i] = .{ .halves = h, .mt = mt, .ks = ks, .fold = f };
+        i += 1;
+    };
+    break :blk vs;
+};
+
+fn wcShared(v: Wc) u32 {
+    const raw = 8 * 4 * v.halves;
+    const stage = v.ks * raw * 4 + v.ks * 16 * v.mt * 32;
+    return @max(4 * stage, 16 * v.mt * 128 * 4);
+}
+
+/// linear.cu dispatch_wc: which linear_wc takes m rows of this weight, or null for the mid-M kernels.
+fn wcPick(w: Weight, m: usize, sk: u32, wk: u32) ?usize {
+    if (w.codebook != .mul1 or (w.halves != 8 and w.halves != 12)) return null;
+    const per_warp = w.k / 16 / sk / wk;
+    if (per_warp % 2 != 0 or m <= 16 or m > 128) return null;
+    const nb = w.n / 128;
+    const fold = sk > 1 and nb >= 32;
+    if (!fold and w.halves == 8 and m > 32 and m <= 40) return null;
+    const four = per_warp % 4 == 0 and !(w.halves == 12 and m > 96);
+    const mt2 = m <= 32 or (m > 64 and m <= 96) or (fold and m > 96 and nb >= 64);
+    const want: Wc = .{ .halves = w.halves, .mt = if (mt2) 2 else 4, .ks = if (four) 4 else 2, .fold = fold };
+    for (wc_variants, 0..) |v, i| if (std.meta.eql(v, want)) return i;
+    return null;
+}
+
+fn mpgIndex(halves: u32, codebook: Codebook, warps: u32, p: u32) ?usize {
+    if (codebook != .mul1) return null;
+    for (mpg_variants, 0..) |v, i| if (v.halves == halves and v.warps == warps and v.p == p) return i;
+    return null;
+}
+
 pub fn variantIndex(halves: u32, codebook: Codebook, warps: u32) ?usize {
     for (variants, 0..) |v, i| {
         if (v.halves == halves and v.codebook == codebook and v.warps == warps) return i;
@@ -98,12 +150,16 @@ pub const Weight = struct {
     codebook: Codebook,
     stride_k: u64,
     stride_nb: u64,
+    sk: u32 = 0, // the K splits and warps a program the family fixed for this shape; 0: plan()'s
+    wk: u32 = 0,
 
     /// Columns [from, from + n) as a view of the stored matrix: whole 128-column blocks, the parent's strides.
     pub fn columns(w: Weight, from: u32, n: u32) !Weight {
         if (from % 128 != 0 or n % 128 != 0 or n == 0 or @as(u64, from) + n > w.n) return error.Exl3Shape;
         var v = w;
         v.n = n;
+        v.sk = 0; // a narrower view plans as its own shape
+        v.wk = 0;
         v.trellis = w.trellis + @as(u64, from / 128) * w.stride_nb * 4;
         v.svh = w.svh + @as(u64, from) * 2;
         if (w.bias != 0) v.bias = w.bias + @as(u64, from) * 2;
@@ -111,11 +167,28 @@ pub const Weight = struct {
     }
 };
 
+/// A weight's launch: the family's fixed (sk, wk) when given, else plan()'s; either is the shape's alone.
+pub fn planOf(w: Weight) struct { sk: u32, wk: u32 } {
+    if (w.sk != 0) return .{ .sk = w.sk, .wk = w.wk };
+    const p = plan(w.k, w.n, 192, 8);
+    return .{ .sk = p.sk, .wk = p.wk };
+}
+
 /// The module and its kernels.
 pub const Linear = struct {
     mod: Module,
     fns: [variants.len]Function,
     rot_in: Function,
+    mpg: [mpg_variants.len]Function = undefined,
+    group: [2]Function = undefined, // tf_exl3_group_8_2_{4,8}: projection groups of 4-bit mul1 layers
+    rot_group: Function = undefined,
+    wc: [wc_variants.len]Function = undefined,
+    /// 17-128 rows of 4- and 6-bit mul1 layers on linear_wc (false: the mid-M kernels; the same bits either way).
+    use_wc: bool = true,
+    /// Programmatic dependent launch for the rotations and the decode-row linears (TF_EXL3_PDL=0: plain launches).
+    pdl: bool = true,
+    /// 17-128 rows on the mid-M kernels where a width has them (false: the linear at every row count, same bits).
+    midm: bool = true,
 
     pub fn load(d: *const Driver) !Linear {
         if (!kernels.available) return error.BuiltWithoutKernels;
@@ -125,6 +198,16 @@ pub const Linear = struct {
         inline for (0..variants.len) |i| {
             l.fns[i] = try l.mod.function(symbol(variants[i].halves, variants[i].codebook, variants[i].warps));
         }
+        inline for (mpg_variants, 0..) |v, i| {
+            l.mpg[i] = try l.mod.function(std.fmt.comptimePrint("tf_exl3_mpg_{d}_{d}_{d}", .{ v.halves, v.warps, v.p }));
+        }
+        inline for (wc_variants, 0..) |v, i| {
+            l.wc[i] = try l.mod.function(std.fmt.comptimePrint("tf_exl3_wc_{d}_{d}_{d}_{d}", .{ v.halves, v.mt, v.ks, @intFromBool(v.fold) }));
+            try l.wc[i].allowDynamicShared(wcShared(v));
+        }
+        l.group = .{ try l.mod.function("tf_exl3_group_8_2_4"), try l.mod.function("tf_exl3_group_8_2_8") };
+        l.rot_group = try l.mod.function("tf_exl3_rot_group");
+        if (platform.c.getenv("TF_EXL3_PDL")) |v| l.pdl = !std.mem.eql(u8, std.mem.span(v), "0");
         return l;
     }
 
@@ -134,13 +217,13 @@ pub const Linear = struct {
 
     /// Shared bytes one call needs: WK * min(m, 8) * 128 floats of warp sums, at most 32 KB.
     pub fn sharedBytes(w: Weight, m: usize) u32 {
-        const p = plan(w.k, w.n, 192, 8);
+        const p = planOf(w);
         return @intCast(p.wk * @min(m, 8) * 128 * 4);
     }
 
     /// Split partials one call needs (fp32 [sk, m, n]) when the plan splits K, else 0.
     pub fn zBytes(w: Weight, m: usize) usize {
-        const p = plan(w.k, w.n, 192, 8);
+        const p = planOf(w);
         return if (p.sk > 1) @as(usize, p.sk) * m * w.n * 4 else 0;
     }
 
@@ -154,7 +237,7 @@ pub const Linear = struct {
         // Shapes the loader admits (linear.py's k % 128 and n % 128, 1..128 rows) are returned errors here too.
         if (m < 1 or m > 128) return error.Exl3Shape;
         if (w.k % 128 != 0 or w.n % 128 != 0) return error.Exl3Shape;
-        const p = plan(w.k, w.n, 192, 8);
+        const p = planOf(w);
         if ((w.k / 16) % (p.sk * p.wk) != 0) return error.Exl3Shape;
         const idx = variantIndex(w.halves, w.codebook, p.wk) orelse return error.UnsupportedExl3Width;
         if (p.sk > 1 and z == 0) return error.Exl3Scratch;
@@ -170,6 +253,7 @@ pub const Linear = struct {
             .grid = .{ .x = @intCast((w.k / 128 + 3) / 4), .y = @intCast(m), .z = 1 },
             .block = .{ .x = 128, .y = 1, .z = 1 },
             .shared = 0,
+            .pdl = l.pdl, // tf_exl3_rot_in waits for the previous kernel's rows itself
         }, s, &r);
 
         var args: launch_.Args = .{};
@@ -187,14 +271,82 @@ pub const Linear = struct {
         args.add(@as(i32, @intCast(p.sk)));
         args.add(@as(i32, @intCast(m)));
         args.add(@as(i32, @intCast(w.k)));
+        // 17-48 rows: 16-row groups; 49-128: 32-row groups, each group's tiles decoded once (linear.cu's dispatch)
+        if (l.midm and l.use_wc) if (wcPick(w, m, p.sk, p.wk)) |wi| {
+            const v = wc_variants[wi];
+            args.add(@as(i32, @intCast(p.wk))); // WcArgs: LinearArgs, then the plan's warps
+            const rg = 16 * v.mt;
+            return launch_.launch(l.wc[wi], .{
+                .grid = .{ .x = @intCast(w.n / 128 * ((m + rg - 1) / rg)), .y = if (v.fold) 1 else p.sk, .z = 1 },
+                .block = .{ .x = 256, .y = 1, .z = 1 },
+                .shared = wcShared(v),
+            }, s, &args);
+        };
+        const pg: u32 = if (m > 48) 2 else 1;
+        if (l.midm and m > 16) if (mpgIndex(w.halves, w.codebook, p.wk, pg)) |mi| {
+            const groups = (m + 16 * pg - 1) / (16 * pg);
+            return launch_.launch(l.mpg[mi], .{
+                .grid = .{ .x = @intCast(w.n / 128 * groups), .y = p.sk, .z = 1 },
+                .block = .{ .x = @intCast(p.wk * 32), .y = 1, .z = 1 },
+                .shared = sharedBytes(w, m),
+            }, s, &args);
+        };
         try launch_.launch(l.fns[idx], .{
             .grid = .{ .x = @intCast(w.n / 128 * p.sk), .y = 1, .z = 1 },
             .block = .{ .x = @intCast(p.wk * 32), .y = 1, .z = 1 },
             .shared = sharedBytes(w, m),
+            .pdl = l.pdl, // prefetches its weights, then waits for the rotation
         }, s, &args);
     }
+
+    /// y_i = x through each of `ws` in one rotation and one launch (a member's own bits); scratch back to back, zeroed.
+    pub fn matmulGroup(l: *const Linear, s: Stream, ws: []const Weight, x: u64, x_dtype: DType, m: usize, ys: []const u64, y_dtype: DType, xh: u64, z: u64, counters: u64) !void {
+        if (!groupable(ws, m) or ys.len != ws.len) return error.Exl3Group;
+        const k = ws[0].k;
+        var rot: RotGroupC = .{ .x = x, .suh = @splat(0), .xh = @splat(0), .x_dtype = @backingInt(x_dtype), .m = @intCast(m), .k = @intCast(k) };
+        var g: GroupArgsC = .{ .l = undefined, .ends = @splat(0) };
+        var zat = z;
+        var cat = counters;
+        var end: i32 = 0;
+        for (ws, ys, 0..) |w, y, i| {
+            const p = planOf(w);
+            if (p.sk > 1 and z == 0) return error.Exl3Scratch;
+            rot.suh[i] = w.suh;
+            rot.xh[i] = xh + i * m * k * 2;
+            g.l[i] = .{ .xh = rot.xh[i], .t = w.trellis, .stride_k = @intCast(w.stride_k), .stride_nb = @intCast(w.stride_nb), .svh = w.svh, .bias = w.bias, .y = y, .z = if (p.sk > 1) zat else 0, .counters = cat, .y_dtype = @backingInt(y_dtype), .n = @intCast(w.n), .sk = @intCast(p.sk), .m = @intCast(m), .k = @intCast(k) };
+            zat += zBytes(w, m);
+            cat += counterBytes(w);
+            end += @intCast(w.n / 128 * p.sk);
+            g.ends[i] = end;
+        }
+        for (ws.len..3) |i| g.ends[i] = end;
+        var ra: launch_.Args = .{};
+        ra.add(rot);
+        try launch_.launch(l.rot_group, .{ .grid = .{ .x = @intCast((k / 128 + 3) / 4), .y = @intCast(m), .z = @intCast(ws.len) }, .block = .{ .x = 128, .y = 1, .z = 1 }, .pdl = l.pdl }, s, &ra);
+        var ga: launch_.Args = .{};
+        ga.add(g);
+        const wk = planOf(ws[0]).wk;
+        try launch_.launch(l.group[if (wk == 4) 0 else 1], .{ .grid = .{ .x = @intCast(end), .y = 1, .z = 1 }, .block = .{ .x = wk * 32, .y = 1, .z = 1 }, .shared = sharedBytes(ws[0], m), .pdl = l.pdl }, s, &ga);
+    }
 };
 
+/// Members one group launch takes: 2 or 3 4-bit mul1 layers of one K and one warp count, at 1 to 16 rows.
+pub fn groupable(ws: []const Weight, m: usize) bool {
+    if (ws.len < 2 or ws.len > 3 or m < 1 or m > 16) return false;
+    const wk = planOf(ws[0]).wk;
+    if (wk != 4 and wk != 8) return false;
+    for (ws) |w| {
+        if (w.halves != 8 or w.codebook != .mul1 or w.k != ws[0].k or planOf(w).wk != wk) return false;
+        if (w.k % 128 != 0 or w.n % 128 != 0 or (w.k / 16) % (planOf(w).sk * wk) != 0) return false;
+    }
+    return true;
+}
+
+/// linear.cu's LinearArgs, GroupArgs and RotGroupArgs as the kernels read them.
+const LinearArgsC = extern struct { xh: u64, t: u64, stride_k: i64, stride_nb: i64, svh: u64, bias: u64, y: u64, z: u64, counters: u64, y_dtype: i32, n: i32, sk: i32, m: i32, k: i32 };
+const GroupArgsC = extern struct { l: [3]LinearArgsC, ends: [3]i32 };
+const RotGroupC = extern struct { x: u64, suh: [3]u64, xh: [3]u64, x_dtype: i32, m: i32, k: i32 };
+
 /// The loaded kernels plus the scratch one stream's dense calls share; prompt rows run in 128-row blocks (forward.py).
 pub const Dense = struct {
     d: *const Driver,
@@ -283,8 +435,26 @@ pub const Dense = struct {
         return k.rows(s, x, w.k, m, y, rows, yd == .f32);
     }
 
+    /// Decode rows of a projection group in one launch where `groupable`, else member by member (the same bits).
+    pub fn decodeGroup(de: *Dense, s: Stream, ws: []const Weight, x: u64, x_dtype: DType, rows: usize, ys: []const u64, y_dtype: DType) !void {
+        if (!groupable(ws, rows)) {
+            for (ws, ys) |w, y| try de.decode(s, w, x, x_dtype, rows, y, y_dtype);
+            return;
+        }
+        var zb: usize = 0;
+        var cb: usize = 0;
+        for (ws) |w| {
+            zb += Linear.zBytes(w, 128);
+            cb += Linear.counterBytes(w);
+        }
+        const z: u64 = if (zb > 0) try de.scratch(&de.z, zb) else 0;
+        const counters = try de.grownCounters(s, cb); // the members' counters back to back
+        const xh = try de.scratch(&de.xh, ws.len * rows * ws[0].k * 2);
+        try de.lin.matmulGroup(s, ws, x, x_dtype, rows, ys, y_dtype, xh, z, counters);
+    }
+
     fn blocks(de: *Dense, s: Stream, w: Weight, x: u64, x_dtype: DType, rows: usize, y: u64, y_dtype: DType) !void {
-        const p = plan(w.k, w.n, 192, 8);
+        const p = planOf(w);
         const z: u64 = if (p.sk > 1) try de.scratch(&de.z, Linear.zBytes(w, 128)) else 0;
         const counters = try de.grownCounters(s, Linear.counterBytes(w));
         const xe: u64 = if (x_dtype == .f32) 4 else 2;
diff --git a/zig/tests/cuda/exl3_plan_bench.zig b/zig/tests/cuda/exl3_plan_bench.zig
new file mode 100644
index 0000000000..d074a7930f
--- /dev/null
+++ b/zig/tests/cuda/exl3_plan_bench.zig
@@ -0,0 +1,271 @@
+//! `tf-cuda-test exl3-plans`: every (K splits, warps) and layout of the 27B's EXL3 shapes timed at 1 and 16 rows.
+const std = @import("std");
+const cuda = @import("cuda");
+const check = @import("check.zig");
+const quant_ref_tests = @import("quant_ref_tests.zig");
+const exl3_tests = @import("exl3_tests.zig");
+const exl3 = cuda.exl3;
+const Gpu = check.Gpu;
+const Rig = quant_ref_tests.RigT;
+
+const Shape = struct { name: []const u8, k: usize, n: usize, halves: u32 };
+
+/// Qwen3.8-27B's projections in turboderp's 4.00bpw pack (6-bit head).
+const shapes = [_]Shape{
+    .{ .name = "gate_up", .k = 5120, .n = 17408, .halves = 8 },
+    .{ .name = "down", .k = 17408, .n = 5120, .halves = 8 },
+    .{ .name = "qkv", .k = 5120, .n = 10240, .halves = 8 },
+    .{ .name = "z", .k = 5120, .n = 6144, .halves = 8 },
+    .{ .name = "out_o", .k = 6144, .n = 5120, .halves = 8 },
+    .{ .name = "q", .k = 5120, .n = 12288, .halves = 8 },
+    .{ .name = "k_v", .k = 5120, .n = 1024, .halves = 8 },
+    .{ .name = "head", .k = 5120, .n = 248320, .halves = 12 },
+};
+
+const splits = [_]u32{ 1, 2, 4, 5, 8, 10, 16, 20, 32, 40 };
+const warps = [_]u32{ 2, 4, 8 };
+const row_counts = [_]usize{ 1, 16 };
+/// Copies of a weight rotated through, so each call reads DRAM rather than L2 (at least this many bytes in all).
+const sweep_bytes = 256 << 20;
+
+pub fn run(gpu: Gpu, only: ?[]const u8) !void {
+    var arena = std.heap.ArenaAllocator.init(gpu.gpa);
+    defer arena.deinit();
+    var s = try cuda.Stream.init(gpu.d, false);
+    defer s.deinit();
+    var prng = std.Random.DefaultPrng.init(5);
+    var r: Rig = .{ .gpu = gpu, .a = arena.allocator(), .s = s, .rng = prng.random() };
+    defer {
+        for (r.bufs.items) |*b| b.free();
+        r.bufs.deinit(gpu.gpa);
+    }
+    var lin = try exl3.Linear.load(gpu.d);
+    defer lin.unload();
+    for (shapes) |sh| {
+        if (only) |o| if (!std.mem.eql(u8, o, sh.name)) continue;
+        try shape(&r, &lin, sh);
+        for (r.bufs.items) |*b| b.free();
+        r.bufs.clearRetainingCapacity();
+    }
+}
+
+fn shape(r: *Rig, lin: *const exl3.Linear, sh: Shape) !void {
+    const words = (sh.k / 16) * (sh.n / 16) * sh.halves * 4;
+    const host = try r.a.alloc(u32, words);
+    for (host) |*v| v.* = r.rng.int(u32);
+    const copies = @min(64, @max(1, sweep_bytes / (words * 4)));
+    var trellis: [64]u64 = undefined;
+    for (trellis[0..copies]) |*t| t.* = try r.dev(std.mem.sliceAsBytes(host));
+    const suh = try r.a.alloc(u16, sh.k);
+    const svh = try r.a.alloc(u16, sh.n);
+    @memset(suh, 0x3C00);
+    @memset(svh, 0x3C00);
+    const dsuh = try r.dev(std.mem.sliceAsBytes(suh));
+    const dsvh = try r.dev(std.mem.sliceAsBytes(svh));
+    const x = try r.dev(std.mem.sliceAsBytes(try r.bfs(16 * sh.k, 1.0)));
+    const y = try r.zeros(16 * sh.n * 2);
+    const xh = try r.zeros(16 * sh.k * 2);
+    const z = try r.zeros(40 * 16 * sh.n * 4);
+    const counters = try exl3_tests.zeroed(r, 8 * (sh.n / 128) * 4);
+    const kt = sh.k / 16;
+    const def = exl3.plan(sh.k, sh.n, 192, 8);
+    for (row_counts) |m| {
+        var best: f64 = std.math.inf(f64);
+        var best_at: [3]u32 = .{ 0, 0, 0 };
+        for ([_]bool{ false, true }) |strips| for (splits) |sk| for (warps) |wk| {
+            if (kt % (sk * wk) != 0 or kt / (sk * wk) < 2) continue;
+            if (sk > 1 and sh.n / 128 * sk > 4096) continue;
+            const st = exl3.strides(sh.k, sh.n, sh.halves, strips);
+            var w: exl3.Weight = .{ .trellis = 0, .suh = dsuh, .svh = dsvh, .k = @intCast(sh.k), .n = @intCast(sh.n), .halves = sh.halves, .codebook = .mul1, .stride_k = st.k, .stride_nb = st.nb, .sk = sk, .wk = wk };
+            for (0..2) |c| {
+                w.trellis = trellis[c % copies];
+                try lin.matmul(r.s, w, x, .bf16, m, y, .bf16, xh, z, counters);
+            }
+            try r.s.synchronize();
+            const reps = @max(copies * 4, 24);
+            const t0 = check.now(r.gpu.io);
+            for (0..reps) |c| {
+                w.trellis = trellis[c % copies];
+                try lin.matmul(r.s, w, x, .bf16, m, y, .bf16, xh, z, counters);
+            }
+            try r.s.synchronize();
+            const us = @as(f64, @floatFromInt(check.now(r.gpu.io) - t0)) / 1e3 / @as(f64, @floatFromInt(reps));
+            const gbs = @as(f64, @floatFromInt(words * 4)) / us / 1e3;
+            const mark: []const u8 = if (!strips and sk == def.sk and wk == def.wk) " (plan)" else "";
+            std.debug.print("PLAN {s} m={d} {s} sk={d} wk={d}: {d:.1} us {d:.0} GB/s{s}\n", .{ sh.name, m, if (strips) "strips" else "stored", sk, wk, us, gbs, mark });
+            if (us < best) {
+                best = us;
+                best_at = .{ @intFromBool(strips), sk, wk };
+            }
+        };
+        std.debug.print("BEST {s} m={d}: {s} sk={d} wk={d} {d:.1} us\n", .{ sh.name, m, if (best_at[0] == 1) "strips" else "stored", best_at[1], best_at[2], best });
+    }
+}
+
+/// The family's fixed plans (cuda_weights_exl3.zig) by shape name, for `midm`.
+const fixed = [_]struct { name: []const u8, sk: u32, wk: u32 }{
+    .{ .name = "gate_up", .sk = 1, .wk = 4 }, .{ .name = "down", .sk = 1, .wk = 8 },  .{ .name = "qkv", .sk = 1, .wk = 4 },
+    .{ .name = "z", .sk = 2, .wk = 4 },       .{ .name = "out_o", .sk = 2, .wk = 8 }, .{ .name = "q", .sk = 1, .wk = 8 },
+    .{ .name = "k_v", .sk = 5, .wk = 8 },     .{ .name = "head", .sk = 1, .wk = 4 },
+};
+
+/// `tf-cuda-test exl3-midm`: the mid-M kernels against the linear at 17-128 rows, byte for byte, and both timed at 128.
+pub fn midm(gpu: Gpu) !void {
+    var arena = std.heap.ArenaAllocator.init(gpu.gpa);
+    defer arena.deinit();
+    var s = try cuda.Stream.init(gpu.d, false);
+    defer s.deinit();
+    var prng = std.Random.DefaultPrng.init(9);
+    var r: Rig = .{ .gpu = gpu, .a = arena.allocator(), .s = s, .rng = prng.random() };
+    defer {
+        for (r.bufs.items) |*b| b.free();
+        r.bufs.deinit(gpu.gpa);
+    }
+    var lin = try exl3.Linear.load(gpu.d);
+    defer lin.unload();
+    var bad: usize = 0;
+    for (shapes, fixed) |sh, f| {
+        const words = (sh.k / 16) * (sh.n / 16) * sh.halves * 4;
+        const host = try r.a.alloc(u32, words);
+        for (host) |*v| v.* = r.rng.int(u32);
+        const suh = try r.a.alloc(u16, sh.k);
+        const svh = try r.a.alloc(u16, sh.n);
+        for (suh) |*v| v.* = @bitCast(@as(f16, @floatCast(r.rng.float(f32) * 2 - 1)));
+        for (svh) |*v| v.* = @bitCast(@as(f16, @floatCast(r.rng.float(f32) * 2 - 1)));
+        const st = exl3.strides(sh.k, sh.n, sh.halves, false);
+        const w: exl3.Weight = .{ .trellis = try r.dev(std.mem.sliceAsBytes(host)), .suh = try r.dev(std.mem.sliceAsBytes(suh)), .svh = try r.dev(std.mem.sliceAsBytes(svh)), .k = @intCast(sh.k), .n = @intCast(sh.n), .halves = sh.halves, .codebook = .mul1, .stride_k = st.k, .stride_nb = st.nb, .sk = f.sk, .wk = f.wk };
+        const x = try r.dev(std.mem.sliceAsBytes(try r.bfs(128 * sh.k, 1.0)));
+        const xh = try r.zeros(128 * sh.k * 2);
+        const z = try r.zeros(exl3.Linear.zBytes(w, 128));
+        const counters = try exl3_tests.zeroed(&r, exl3.Linear.counterBytes(w));
+        const ya = try r.zeros(128 * sh.n * 2);
+        const yb = try r.zeros(128 * sh.n * 2);
+        for ([_]usize{ 17, 24, 32, 33, 48, 49, 64, 100, 128 }) |m| {
+            lin.midm = false;
+            try lin.matmul(r.s, w, x, .bf16, m, ya, .bf16, xh, z, counters);
+            lin.midm = true;
+            try lin.matmul(r.s, w, x, .bf16, m, yb, .bf16, xh, z, counters);
+            const a = try r.back(ya, m * sh.n * 2);
+            const b = try r.back(yb, m * sh.n * 2);
+            if (!std.mem.eql(u8, a, b)) {
+                bad += 1;
+                std.debug.print("FAIL exl3.midm {s} m={d}: the mid-M bits differ\n", .{ sh.name, m });
+            }
+        }
+        // the mid-M kernels alone against the linear too, where linear_wc would take the rows
+        lin.use_wc = false;
+        for ([_]usize{ 24, 48, 64, 128 }) |m| {
+            lin.midm = false;
+            try lin.matmul(r.s, w, x, .bf16, m, ya, .bf16, xh, z, counters);
+            lin.midm = true;
+            try lin.matmul(r.s, w, x, .bf16, m, yb, .bf16, xh, z, counters);
+            if (!std.mem.eql(u8, try r.back(ya, m * sh.n * 2), try r.back(yb, m * sh.n * 2))) {
+                bad += 1;
+                std.debug.print("FAIL exl3.midm {s} m={d}: the mpg bits differ\n", .{ sh.name, m });
+            }
+        }
+        lin.use_wc = true;
+        var us: [3]f64 = undefined;
+        for (0..3) |mode| {
+            lin.midm = mode >= 1;
+            lin.use_wc = mode == 2;
+            try lin.matmul(r.s, w, x, .bf16, 128, ya, .bf16, xh, z, counters);
+            try r.s.synchronize();
+            const t0 = check.now(r.gpu.io);
+            for (0..10) |_| try lin.matmul(r.s, w, x, .bf16, 128, ya, .bf16, xh, z, counters);
+            try r.s.synchronize();
+            us[mode] = @as(f64, @floatFromInt(check.now(r.gpu.io) - t0)) / 1e4;
+        }
+        lin.use_wc = true;
+        std.debug.print("RESULT exl3.midm {s} m=128: linear {d:.0} us, mid-M {d:.0} us, wc {d:.0} us\n", .{ sh.name, us[0], us[1], us[2] });
+        for (r.bufs.items) |*b| b.free();
+        r.bufs.clearRetainingCapacity();
+    }
+    bad += try groups(&r, &lin);
+    try check.expect(bad == 0, "exl3.midm: {d} row counts differ", .{bad});
+    check.pass("exl3.midm", .{});
+}
+
+/// `tf-cuda-test exl3-shapes [rows]`: each 27B shape once at its fixed plan, 1 and 16 rows, for a profiler to attach.
+pub fn profileShapes(gpu: Gpu, rows_arg: ?[]const u8) !void {
+    var arena = std.heap.ArenaAllocator.init(gpu.gpa);
+    defer arena.deinit();
+    var s = try cuda.Stream.init(gpu.d, false);
+    defer s.deinit();
+    var prng = std.Random.DefaultPrng.init(3);
+    var r: Rig = .{ .gpu = gpu, .a = arena.allocator(), .s = s, .rng = prng.random() };
+    defer {
+        for (r.bufs.items) |*b| b.free();
+        r.bufs.deinit(gpu.gpa);
+    }
+    var lin = try exl3.Linear.load(gpu.d);
+    defer lin.unload();
+    const ms: []const usize = if (rows_arg) |a| &.{try std.fmt.parseInt(usize, a, 10)} else &.{ 1, 16 };
+    for (shapes, fixed) |sh, f| {
+        const words = (sh.k / 16) * (sh.n / 16) * sh.halves * 4;
+        const host = try r.a.alloc(u32, words);
+        for (host) |*v| v.* = r.rng.int(u32);
+        const suh = try r.a.alloc(u16, sh.k);
+        const svh = try r.a.alloc(u16, sh.n);
+        @memset(suh, 0x3C00);
+        @memset(svh, 0x3C00);
+        const st = exl3.strides(sh.k, sh.n, sh.halves, false);
+        const w: exl3.Weight = .{ .trellis = try r.dev(std.mem.sliceAsBytes(host)), .suh = try r.dev(std.mem.sliceAsBytes(suh)), .svh = try r.dev(std.mem.sliceAsBytes(svh)), .k = @intCast(sh.k), .n = @intCast(sh.n), .halves = sh.halves, .codebook = .mul1, .stride_k = st.k, .stride_nb = st.nb, .sk = f.sk, .wk = f.wk };
+        const x = try r.dev(std.mem.sliceAsBytes(try r.bfs(128 * sh.k, 1.0)));
+        const xh = try r.zeros(128 * sh.k * 2);
+        const z = try r.zeros(exl3.Linear.zBytes(w, 128) + 16);
+        const counters = try exl3_tests.zeroed(&r, exl3.Linear.counterBytes(w));
+        const y = try r.zeros(128 * sh.n * 2);
+        for (ms) |m| {
+            std.debug.print("SHAPE {s} m={d} sk={d} wk={d}\n", .{ sh.name, m, f.sk, f.wk });
+            for (0..2) |_| try lin.matmul(r.s, w, x, .bf16, m, y, .bf16, xh, z, counters);
+            try r.s.synchronize();
+        }
+        for (r.bufs.items) |*b| b.free();
+        r.bufs.clearRetainingCapacity();
+    }
+}
+
+/// The 27B's projection groups (q k v, gate up, qkv z) in one launch against each member's own, byte for byte.
+fn groups(r: *Rig, lin: *const exl3.Linear) !usize {
+    const sets = [_][]const usize{ &.{ 5, 6, 6 }, &.{ 0, 0 }, &.{ 2, 3 } }; // indices into shapes / fixed
+    var bad: usize = 0;
+    for (sets) |set| {
+        var ws: [3]exl3.Weight = undefined;
+        var ya: [3]u64 = undefined;
+        var yb: [3]u64 = undefined;
+        for (set, 0..) |si, i| {
+            const sh = shapes[si];
+            const f = fixed[si];
+            const words = (sh.k / 16) * (sh.n / 16) * sh.halves * 4;
+            const host = try r.a.alloc(u32, words);
+            for (host) |*v| v.* = r.rng.int(u32);
+            const suh = try r.a.alloc(u16, sh.k);
+            const svh = try r.a.alloc(u16, sh.n);
+            for (suh) |*v| v.* = @bitCast(@as(f16, @floatCast(r.rng.float(f32) * 2 - 1)));
+            for (svh) |*v| v.* = @bitCast(@as(f16, @floatCast(r.rng.float(f32) * 2 - 1)));
+            const st = exl3.strides(sh.k, sh.n, sh.halves, false);
+            ws[i] = .{ .trellis = try r.dev(std.mem.sliceAsBytes(host)), .suh = try r.dev(std.mem.sliceAsBytes(suh)), .svh = try r.dev(std.mem.sliceAsBytes(svh)), .k = @intCast(sh.k), .n = @intCast(sh.n), .halves = sh.halves, .codebook = .mul1, .stride_k = st.k, .stride_nb = st.nb, .sk = f.sk, .wk = f.wk };
+            ya[i] = try r.zeros(16 * sh.n * 2);
+            yb[i] = try r.zeros(16 * sh.n * 2);
+        }
+        const g = ws[0..set.len];
+        const k = g[0].k;
+        const x = try r.dev(std.mem.sliceAsBytes(try r.bfs(16 * k, 1.0)));
+        const xh = try r.zeros(3 * 16 * k * 2);
+        const z = try r.zeros(3 * 40 * 16 * 12288 * 4);
+        const counters = try exl3_tests.zeroed(r, 3 * 8 * (248320 / 128) * 4);
+        for ([_]usize{ 1, 7, 16 }) |m| {
+            for (g, 0..) |w, i| try lin.matmul(r.s, w, x, .bf16, m, ya[i], .bf16, xh, z, counters);
+            try lin.matmulGroup(r.s, g, x, .bf16, m, yb[0..set.len], .bf16, xh, z, counters);
+            for (g, 0..) |w, i| if (!std.mem.eql(u8, try r.back(ya[i], m * w.n * 2), try r.back(yb[i], m * w.n * 2))) {
+                bad += 1;
+                std.debug.print("FAIL exl3.group set {d} member {d} m={d}: the group launch's bits differ\n", .{ set[0], i, m });
+            };
+        }
+        for (r.bufs.items) |*b| b.free();
+        r.bufs.clearRetainingCapacity();
+    }
+    if (bad == 0) check.pass("exl3.group", .{});
+    return bad;
+}
diff --git a/zig/tests/cuda/main.zig b/zig/tests/cuda/main.zig
index 786bd2ee0f..556e2a0bf2 100644
--- a/zig/tests/cuda/main.zig
+++ b/zig/tests/cuda/main.zig
@@ -41,6 +41,7 @@ const nvfp4_tests = @import("nvfp4_tests.zig");
 const modelopt_tests = @import("modelopt_tests.zig");
 const exl3_tests = @import("exl3_tests.zig");
 const exl3_tp2_tests = @import("exl3_tp2_tests.zig");
+const exl3_plan_bench = @import("exl3_plan_bench.zig");
 const exl3_candidate_tests = @import("exl3_candidate_tests.zig");
 const experts3_tests = @import("experts3_tests.zig");
 const moe_tests = @import("moe_tests.zig");
@@ -224,6 +225,9 @@ const usage =
     \\  moe-glue                  sigmoid top-k routing and the weighted combine against float64 references (synthetic)
     \\  dense-rows                bf16 rows times weight rows (routers, heads) against float64 references (synthetic)
     \\  exl3 [dir]                EXL3 decode-row linear: float64 checks, row invariance, the Python engine's fixture (dir none skips it)
+    \\  exl3-shapes [rows]        the 27B's EXL3 shapes once each with the fixed plans (1 and 16 rows), for ncu
+    \\  exl3-midm                 the mid-M kernels, linear_wc and the projection groups against the linear, byte for byte
+    \\  exl3-plans [shape]        the 27B's EXL3 shapes: every (K splits, warps) and layout timed at 1 and 16 rows
     \\  exl3-tp2 <rank> <host:port> [dir]  EXL3 tensor parallel: rank shards vs the Python engine's, all-gathered (dir none skips them)
     \\  exl3-tiles [DIR]          EXL3's tile candidate: codebooks and tiles bit-equal, FP64, rows through qlinear and Dense
     \\  train-pack MODEL EXAMPLES [COUNT]   training passes holding several sequences against one a pass: differences, time
@@ -392,6 +396,9 @@ fn run(gpu: check.Gpu, cmd: []const u8, rest: []const [:0]const u8) !void {
     if (std.mem.eql(u8, cmd, "exlr8-bench")) return exlr8_tests.bench(gpu, try arg(rest, 0), if (rest.len > 1) rest[1] else "home", if (rest.len > 2) try std.fmt.parseInt(u16, rest[2], 10) else 40, if (rest.len > 3) try std.fmt.parseInt(usize, rest[3], 10) else 4);
     if (std.mem.eql(u8, cmd, "exl3")) return exl3_tests.run(gpu, if (rest.len > 0) rest[0] else "none");
     if (std.mem.eql(u8, cmd, "exl3-tiles")) return exl3_candidate_tests.run(gpu, if (rest.len > 0) rest[0] else null);
+    if (std.mem.eql(u8, cmd, "exl3-shapes")) return exl3_plan_bench.profileShapes(gpu, if (rest.len > 0) rest[0] else null);
+    if (std.mem.eql(u8, cmd, "exl3-midm")) return exl3_plan_bench.midm(gpu);
+    if (std.mem.eql(u8, cmd, "exl3-plans")) return exl3_plan_bench.run(gpu, if (rest.len > 0) rest[0] else null);
     if (std.mem.eql(u8, cmd, "exl3-tp2")) return exl3_tp2_tests.run(gpu, try arg(rest, 0), try arg(rest, 1), if (rest.len > 2) rest[2] else "none");
     if (std.mem.eql(u8, cmd, "sample")) return sample_tests.draws(gpu);
     if (std.mem.eql(u8, cmd, "topk")) return sample_tests.topk(gpu);

From ebdc98e9cd6ccafda8c4ef55059b5fb2568c865f Mon Sep 17 00:00:00 2001
From: BHCC2025 <241330250+BHCC2025@users.noreply.github.com>
Date: Tue, 27 Oct 2026 09:06:49 -0500
Subject: [PATCH 2/8] qwen3_5 cuda: EXL3 decode groups and fixed plans for the
 27B's shapes (same bits)

At decode rows an EXL3 engine's projMany puts its EXL3 members in one cuda.exl3 group launch where groupable (q/k/v,
qkv/z, and gate/up through projMany); each member's bits are its own launch's. Dense.reserveGroup sizes a group's
scratch at load, before any graph capture, and Dense.reserve sizes split-K scratch by planOf, so a fixed plan's
splits are reserved too. The loader fixes (K splits, warps) for the Qwen3.8-27B projection shapes.
---
 zig/src/cuda/exl3.zig                         | 16 ++++++++-
 .../families/qwen3_5/cuda_exl3_weights.zig    | 35 +++++++++++++++++++
 zig/src/families/qwen3_5/cuda_forward.zig     |  8 +++--
 .../families/qwen3_5/cuda_projection_many.zig | 25 +++++++++++++
 4 files changed, 81 insertions(+), 3 deletions(-)

diff --git a/zig/src/cuda/exl3.zig b/zig/src/cuda/exl3.zig
index 3920548ae0..bfeee976a1 100644
--- a/zig/src/cuda/exl3.zig
+++ b/zig/src/cuda/exl3.zig
@@ -384,7 +384,7 @@ pub const Dense = struct {
 
     /// Grow every scratch buffer to what 128 rows of `w` need on `s`, so a call in a graph capture allocates nothing.
     pub fn reserve(de: *Dense, s: Stream, w: Weight) !void {
-        if (plan(w.k, w.n, 192, 8).sk > 1) _ = try de.scratch(&de.z, Linear.zBytes(w, 128));
+        if (planOf(w).sk > 1) _ = try de.scratch(&de.z, Linear.zBytes(w, 128)); // a fixed plan's splits too
         _ = try de.grownCounters(s, Linear.counterBytes(w));
         _ = try de.scratch(&de.xh, 128 * w.k * 2);
     }
@@ -397,6 +397,20 @@ pub const Dense = struct {
         return ptr;
     }
 
+    /// `reserve` for a group `decodeGroup` launches at once: its members' scratch side by side, at 16 rows.
+    pub fn reserveGroup(de: *Dense, s: Stream, ws: []const Weight) !void {
+        if (!groupable(ws, 16)) return;
+        var zb: usize = 0;
+        var cb: usize = 0;
+        for (ws) |w| {
+            zb += Linear.zBytes(w, 128);
+            cb += Linear.counterBytes(w);
+        }
+        if (zb > 0) _ = try de.scratch(&de.z, zb);
+        _ = try de.grownCounters(s, cb);
+        _ = try de.scratch(&de.xh, ws.len * 16 * ws[0].k * 2);
+    }
+
     /// Decode rows: one call (the kernel takes 1..128), any count above in 128-row blocks.
     pub fn decode(de: *Dense, s: Stream, w: Weight, x: u64, x_dtype: DType, rows: usize, y: u64, y_dtype: DType) !void {
         if (de.implementation == .tiles) return de.tiles(s, w, x, x_dtype, rows, y, y_dtype, false);
diff --git a/zig/src/families/qwen3_5/cuda_exl3_weights.zig b/zig/src/families/qwen3_5/cuda_exl3_weights.zig
index de931dd992..7e22a6cd10 100644
--- a/zig/src/families/qwen3_5/cuda_exl3_weights.zig
+++ b/zig/src/families/qwen3_5/cuda_exl3_weights.zig
@@ -126,11 +126,30 @@ fn f32vec(s: *Sink, ck: *core.Checkpoint, comptime fmt: []const u8, args: anytyp
 const admit = @import("cuda_exl3_admit.zig");
 const group = admit.group;
 
+/// (K splits, warps) per Qwen3.8-27B projection (K, N, halves), chosen with tf-cuda-test exl3-plans.
+const plans = [_]struct { k: usize, n: usize, halves: u32, sk: u32, wk: u32 }{
+    .{ .k = 5120, .n = 17408, .halves = 8, .sk = 1, .wk = 4 }, // gate, up
+    .{ .k = 17408, .n = 5120, .halves = 8, .sk = 1, .wk = 8 }, // down
+    .{ .k = 5120, .n = 10240, .halves = 8, .sk = 1, .wk = 4 }, // in_proj_qkv
+    .{ .k = 5120, .n = 6144, .halves = 8, .sk = 2, .wk = 4 }, // in_proj_z
+    .{ .k = 6144, .n = 5120, .halves = 8, .sk = 2, .wk = 8 }, // out_proj, o_proj
+    .{ .k = 5120, .n = 12288, .halves = 8, .sk = 1, .wk = 8 }, // q_proj (8 warps: one group launch with k and v)
+    .{ .k = 5120, .n = 1024, .halves = 8, .sk = 5, .wk = 8 }, // k_proj, v_proj
+    .{ .k = 5120, .n = 248320, .halves = 12, .sk = 1, .wk = 4 }, // lm_head (6-bit)
+};
+
+/// A shape's fixed plan, so every row count keeps one reduction; other shapes take plan()'s.
+fn tuned(k: usize, n: usize, halves: u32) ?[2]u32 {
+    for (plans) |p| if (p.k == k and p.n == n and p.halves == halves) return .{ p.sk, p.wk };
+    return null;
+}
+
 /// An n-by-k EXL3 projection, its trellis kept as stored (Dense reads the stored layout).
 fn dense(s: *Sink, kit: ?*Kit, ck: *core.Checkpoint, prefix: []const u8, mod: []const u8, n: usize, k: usize) !Proj {
     var buf: [200]u8 = undefined;
     const gr = try admit.denseGroup(ck, try name(&buf, "{s}{s}", .{ prefix, mod }), n, k);
     const st = cuda.exl3.strides(k, n, gr.halves, false);
+    const fixed = tuned(k, n, gr.halves);
     const w: cuda.exl3.Weight = .{
         .trellis = try s.put(gr.trellis.bytes),
         .suh = try s.put(gr.suh.bytes),
@@ -141,7 +160,10 @@ fn dense(s: *Sink, kit: ?*Kit, ck: *core.Checkpoint, prefix: []const u8, mod: []
         .codebook = .mul1,
         .stride_k = st.k,
         .stride_nb = st.nb,
+        .sk = if (fixed) |f| f[0] else 0,
+        .wk = if (fixed) |f| f[1] else 0,
     };
+    if (cuda.exl3.variantIndex(w.halves, w.codebook, cuda.exl3.planOf(w).wk) == null) return error.UnsupportedQwenQuantization;
     if (kit) |x| try x.dense.reserve(s.e.stream, w);
     return .{ .x3 = w };
 }
@@ -277,6 +299,7 @@ fn all(s: *Sink, ck: *core.Checkpoint) !void {
                 .k_norm = try norm1(s, ck, "{s}self_attn.k_norm.weight", .{p}, geometry.head_dim),
             } };
         }
+        if (kit) |k| try reserveGroups(s.e.stream, k, l.*);
         l.mlp = if (g.moe()) .{ .moe = .{ .router = try router(s, ck, p), .experts = try experts(s, ck, p) } } else .{ .dense = .{
             .gate = try dense(s, kit, ck, p, "mlp.gate_proj", g.intermediate, g.hidden),
             .up = try dense(s, kit, ck, p, "mlp.up_proj", g.intermediate, g.hidden),
@@ -285,6 +308,18 @@ fn all(s: *Sink, ck: *core.Checkpoint) !void {
     }
 }
 
+/// Grouped decode launches (q/k/v, qkv/z, gate/up) need their members' scratch together before any graph capture.
+fn reserveGroups(st: cuda.Stream, k: *Kit, l: engine.Layer) !void {
+    switch (l.mixer) {
+        .delta => |m| try k.dense.reserveGroup(st, &.{ m.qkv.x3, m.z.x3 }),
+        .attention => |m| try k.dense.reserveGroup(st, &.{ m.q.x3, m.k.x3, m.v.x3 }),
+    }
+    switch (l.mlp) {
+        .dense => |m| try k.dense.reserveGroup(st, &.{ m.gate.x3, m.up.x3 }),
+        .moe => {},
+    }
+}
+
 /// The arena bytes the weights take (the scratch is added by the caller).
 pub fn measure(e: *Engine, ck: *core.Checkpoint) !usize {
     var s: Sink = .{ .e = e, .measure = true };
diff --git a/zig/src/families/qwen3_5/cuda_forward.zig b/zig/src/families/qwen3_5/cuda_forward.zig
index e0df39ace6..3763a8d3d7 100644
--- a/zig/src/families/qwen3_5/cuda_forward.zig
+++ b/zig/src/families/qwen3_5/cuda_forward.zig
@@ -97,8 +97,12 @@ pub fn body(e: *E.Engine, shape: anytype) !void {
         if (e.learning) |k| try k.post(i, e.h, e.x, total);
         switch (l.mlp) {
             .dense => |m| {
-                try e.projAgain(m.gate, e.x, e.gate, total, prompt);
-                try e.projAgain(m.up, e.x, e.up, total, prompt);
+                if (m.gate == .x3) {
+                    try e.projMany(&.{ m.gate, m.up }, e.x, &.{ e.gate, e.up }, total, prompt); // one EXL3 group launch
+                } else {
+                    try e.projAgain(m.gate, e.x, e.gate, total, prompt);
+                    try e.projAgain(m.up, e.x, e.up, total, prompt);
+                }
                 if (e.learning) |k| try k.mlp(i, e.gate, e.up, total);
                 const n: c_longlong = @intCast(total * g.intermediate);
                 try e.run(e.fns.mlp_act, .{ (total * g.intermediate + 1023) / 1024, 1, 1 }, .{ 256, 1, 1 }, .{ e.gate, e.up, e.act, n, t_params, @as(c_int, @intCast(g.intermediate)), sums });
diff --git a/zig/src/families/qwen3_5/cuda_projection_many.zig b/zig/src/families/qwen3_5/cuda_projection_many.zig
index 5844239598..c82a78aa19 100644
--- a/zig/src/families/qwen3_5/cuda_projection_many.zig
+++ b/zig/src/families/qwen3_5/cuda_projection_many.zig
@@ -7,6 +7,7 @@ const Proj = engine.Proj;
 
 pub fn projectMany(e: *Engine, ls: []const Proj, x: u64, xs: u64, outs: []const u64, rows: usize, prompt: bool, params: u64, sums: bool, allow_fused_prompt: bool) !void {
     if (ls.len == 0 or ls.len != outs.len) return error.UnexpectedTensor;
+    if (e.kit != null and !(prompt and rows > 16)) return exl3Group(e, ls, x, xs, outs, rows, prompt, params, sums);
     const A = cuda.qlinear.Affine4Kernels;
     const fused_prompt = allow_fused_prompt and prompt and rows > A.max_rows and qwen38PromptShape(ls, rows) and !e.int8.on and e.affine.pdl and blk: {
         const k = ls[0].inputs();
@@ -73,3 +74,27 @@ test "Qwen3.8 GB10 large prompt tile admits only screened projection shapes" {
 fn affine(n: usize) Proj {
     return .{ .a4 = .{ .w = 0, .s = 0, .b = 0, .n = n, .k = 5120, .npad = n } };
 }
+
+/// Decode rows of an EXL3 engine: groupable EXL3 members in one launch (each keeps its own bits), the rest one by one.
+fn exl3Group(e: *Engine, ls: []const Proj, x: u64, xs: u64, outs: []const u64, rows: usize, prompt: bool, params: u64, sums: bool) !void {
+    var ws: [3]cuda.exl3.Weight = undefined;
+    var ys: [3]u64 = undefined;
+    var n: usize = 0;
+    for (ls, outs) |l, out| if (l == .x3 and n < ws.len) {
+        ws[n] = l.x3;
+        ys[n] = out;
+        n += 1;
+    };
+    const grouped = n >= 2 and cuda.exl3.groupable(ws[0..n], rows);
+    if (grouped) try e.kit.?.dense.decodeGroup(e.stream, ws[0..n], x, .bf16, rows, ys[0..n], .bf16);
+    var first = true;
+    var taken: usize = 0;
+    for (ls, outs) |l, out| {
+        if (grouped and l == .x3 and taken < n) {
+            taken += 1;
+            continue;
+        }
+        try e.project(l, x, xs, out, rows, prompt, params, sums and first);
+        first = false;
+    }
+}

From 239e89ee33f782ff777566c0c5fd211c07f9ba3c Mon Sep 17 00:00:00 2001
From: BHCC2025 <241330250+BHCC2025@users.noreply.github.com>
Date: Tue, 27 Oct 2026 10:28:11 -0500
Subject: [PATCH 3/8] exl3 cuda: builds for sm_80-89, the group kernels named
 in source, contract records for the new launches

pdl_wait and pdl_trigger compile to nothing below sm_90, and Linear.load turns dependent launches on only for
compute capability 9.0 and newer. tf_exl3_group_8_2_4 and _8 are written out, so kernel_symbols finds their names.
The registry takes the 2-warp linears, the mid-M kernels, linear_wc, the groups and rot_group (exl3_dense_records.zig),
with usage from nvcc 13.0.88 fatbins for sm_80, 86, 89, 120 and 121, and the existing exl3_linear records refreshed.
---
 zig/kernels/cuda/exl3_linear.cu              |  32 +++--
 zig/src/core/contract/exl3_dense_records.zig |  77 ++++++++++++
 zig/src/core/contract/registry.zig           | 122 +++++++++----------
 zig/src/cuda/exl3.zig                        |  10 ++
 4 files changed, 170 insertions(+), 71 deletions(-)
 create mode 100644 zig/src/core/contract/exl3_dense_records.zig

diff --git a/zig/kernels/cuda/exl3_linear.cu b/zig/kernels/cuda/exl3_linear.cu
index 42535726a4..4ac79c8015 100644
--- a/zig/kernels/cuda/exl3_linear.cu
+++ b/zig/kernels/cuda/exl3_linear.cu
@@ -101,8 +101,16 @@ __device__ __forceinline__ void step_lane_words(const uint32_t (&raw)[8 * lane_w
 }
 
 // Programmatic dependent launch: wait for the previous kernel's writes here (a no-op without it); trigger the next.
-__device__ __forceinline__ void pdl_wait() { asm volatile("griddepcontrol.wait;\n" ::: "memory"); }
-__device__ __forceinline__ void pdl_trigger() { asm volatile("griddepcontrol.launch_dependents;\n" ::: "memory"); }
+__device__ __forceinline__ void pdl_wait() {
+#if __CUDA_ARCH__ >= 900
+    asm volatile("griddepcontrol.wait;\n" ::: "memory");
+#endif
+}
+__device__ __forceinline__ void pdl_trigger() {
+#if __CUDA_ARCH__ >= 900
+    asm volatile("griddepcontrol.launch_dependents;\n" ::: "memory");
+#endif
+}
 
 // xh = fp16(((x * suh) @ H) / sqrt(128)): one warp per 128 columns, one block of 4 warps per 512.
 __device__ __forceinline__ void rot_in_body(const void* x, int x_dtype, const half* suh, half* xh, int kdim) {
@@ -460,14 +468,18 @@ struct GroupArgs {
     LinearArgs l[3];
     int ends[3];
 };
-#define TF_EXL3_GROUP(K2, CB, WK)                                                        \
-    extern "C" __global__ void __launch_bounds__((WK) * 32)                              \
-        tf_exl3_group_##K2##_##CB##_##WK(tf_exl3::GroupArgs g) {                         \
-        const int b = blockIdx.x, i = b < g.ends[0] ? 0 : b < g.ends[1] ? 1 : 2;         \
-        tf_exl3::linear_body<K2, CB, WK>(g.l[i], b - (i ? g.ends[i - 1] : 0));           \
-    }
-TF_EXL3_GROUP(8, 2, 4) TF_EXL3_GROUP(8, 2, 8)
-#undef TF_EXL3_GROUP
+// Member i of a group runs linear_body on its own blocks, exactly as its own launch would.
+__device__ __forceinline__ int group_member(const tf_exl3::GroupArgs& g, int b) {
+    return b < g.ends[0] ? 0 : b < g.ends[1] ? 1 : 2;
+}
+extern "C" __global__ void __launch_bounds__(128) tf_exl3_group_8_2_4(tf_exl3::GroupArgs g) {
+    const int b = blockIdx.x, i = group_member(g, b);
+    tf_exl3::linear_body<8, 2, 4>(g.l[i], b - (i ? g.ends[i - 1] : 0));
+}
+extern "C" __global__ void __launch_bounds__(256) tf_exl3_group_8_2_8(tf_exl3::GroupArgs g) {
+    const int b = blockIdx.x, i = group_member(g, b);
+    tf_exl3::linear_body<8, 2, 8>(g.l[i], b - (i ? g.ends[i - 1] : 0));
+}
 #undef TF_EXL3_CODEBOOK
 #undef TF_EXL3_WIDTHS
 }  // namespace tf_exl3
diff --git a/zig/src/core/contract/exl3_dense_records.zig b/zig/src/core/contract/exl3_dense_records.zig
new file mode 100644
index 0000000000..c5ab926970
--- /dev/null
+++ b/zig/src/core/contract/exl3_dense_records.zig
@@ -0,0 +1,77 @@
+//! EXL3 on Dense (#260): the 2-warp linears, the mid-M kernels, linear_wc, projection groups and their rotation.
+const registry = @import("registry.zig");
+pub const cuda = [_]registry.CudaRecord{
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_2_2_2", .symbol = "tf_exl3_linear_2_2_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 162, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 162, .static = 128, .bound = 64 }, .{ .regs = 162, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_4_2_2", .symbol = "tf_exl3_linear_4_2_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 162, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 162, .static = 128, .bound = 64 }, .{ .regs = 162, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_6_2_2", .symbol = "tf_exl3_linear_6_2_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 160, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_8_2_2", .symbol = "tf_exl3_linear_8_2_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 162, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 162, .static = 128, .bound = 64 }, .{ .regs = 162, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_10_2_2", .symbol = "tf_exl3_linear_10_2_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 186, .static = 128, .bound = 64 }, .{ .regs = 187, .static = 128, .bound = 64 }, .{ .regs = 187, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_12_2_2", .symbol = "tf_exl3_linear_12_2_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 186, .static = 128, .bound = 64 }, .{ .regs = 187, .static = 128, .bound = 64 }, .{ .regs = 187, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_14_2_2", .symbol = "tf_exl3_linear_14_2_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 167, .static = 128, .bound = 64 }, .{ .regs = 167, .static = 128, .bound = 64 }, .{ .regs = 167, .static = 128, .bound = 64 }, .{ .regs = 127, .static = 128, .bound = 64 }, .{ .regs = 127, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_16_2_2", .symbol = "tf_exl3_linear_16_2_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 128, .static = 128, .bound = 64 }, .{ .regs = 128, .static = 128, .bound = 64 }, .{ .regs = 128, .static = 128, .bound = 64 }, .{ .regs = 126, .static = 128, .bound = 64 }, .{ .regs = 126, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_3_2_2", .symbol = "tf_exl3_linear_3_2_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 160, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_5_2_2", .symbol = "tf_exl3_linear_5_2_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 160, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 166, .static = 128, .bound = 64 }, .{ .regs = 166, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_7_2_2", .symbol = "tf_exl3_linear_7_2_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 168, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_9_2_2", .symbol = "tf_exl3_linear_9_2_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 186, .static = 128, .bound = 64 }, .{ .regs = 187, .static = 128, .bound = 64 }, .{ .regs = 187, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_11_2_2", .symbol = "tf_exl3_linear_11_2_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 186, .static = 128, .bound = 64 }, .{ .regs = 187, .static = 128, .bound = 64 }, .{ .regs = 187, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_13_2_2", .symbol = "tf_exl3_linear_13_2_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 167, .static = 128, .bound = 64 }, .{ .regs = 167, .static = 128, .bound = 64 }, .{ .regs = 167, .static = 128, .bound = 64 }, .{ .regs = 127, .static = 128, .bound = 64 }, .{ .regs = 127, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_2_1_2", .symbol = "tf_exl3_linear_2_1_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 162, .static = 128, .bound = 64 }, .{ .regs = 162, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_4_1_2", .symbol = "tf_exl3_linear_4_1_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 162, .static = 128, .bound = 64 }, .{ .regs = 162, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_6_1_2", .symbol = "tf_exl3_linear_6_1_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 160, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 166, .static = 128, .bound = 64 }, .{ .regs = 166, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_8_1_2", .symbol = "tf_exl3_linear_8_1_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 162, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 162, .static = 128, .bound = 64 }, .{ .regs = 162, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_10_1_2", .symbol = "tf_exl3_linear_10_1_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 168, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_12_1_2", .symbol = "tf_exl3_linear_12_1_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 168, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_14_1_2", .symbol = "tf_exl3_linear_14_1_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 165, .static = 128, .bound = 64 }, .{ .regs = 165, .static = 128, .bound = 64 }, .{ .regs = 165, .static = 128, .bound = 64 }, .{ .regs = 127, .static = 128, .bound = 64 }, .{ .regs = 127, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_16_1_2", .symbol = "tf_exl3_linear_16_1_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 128, .static = 128, .bound = 64 }, .{ .regs = 128, .static = 128, .bound = 64 }, .{ .regs = 128, .static = 128, .bound = 64 }, .{ .regs = 128, .static = 128, .bound = 64 }, .{ .regs = 128, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_2_0_2", .symbol = "tf_exl3_linear_2_0_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 162, .static = 128, .bound = 64 }, .{ .regs = 162, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_4_0_2", .symbol = "tf_exl3_linear_4_0_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 162, .static = 128, .bound = 64 }, .{ .regs = 162, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_6_0_2", .symbol = "tf_exl3_linear_6_0_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 160, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 166, .static = 128, .bound = 64 }, .{ .regs = 166, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_8_0_2", .symbol = "tf_exl3_linear_8_0_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 162, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 164, .static = 128, .bound = 64 }, .{ .regs = 162, .static = 128, .bound = 64 }, .{ .regs = 162, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_10_0_2", .symbol = "tf_exl3_linear_10_0_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 168, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_12_0_2", .symbol = "tf_exl3_linear_12_0_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 168, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 }, .{ .regs = 167, .static = 128, .bound = 64 }, .{ .regs = 167, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_14_0_2", .symbol = "tf_exl3_linear_14_0_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 167, .static = 128, .bound = 64 }, .{ .regs = 167, .static = 128, .bound = 64 }, .{ .regs = 167, .static = 128, .bound = 64 }, .{ .regs = 128, .static = 128, .bound = 64 }, .{ .regs = 128, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_16_0_2", .symbol = "tf_exl3_linear_16_0_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 128, .static = 128, .bound = 64 }, .{ .regs = 128, .static = 128, .bound = 64 }, .{ .regs = 128, .static = 128, .bound = 64 }, .{ .regs = 128, .static = 128, .bound = 64 }, .{ .regs = 128, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_mpg_6_2_1", .symbol = "tf_exl3_mpg_6_2_1", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 126, .static = 128, .bound = 64 }, .{ .regs = 128, .static = 128, .bound = 64 }, .{ .regs = 128, .static = 128, .bound = 64 }, .{ .regs = 143, .static = 128, .bound = 64 }, .{ .regs = 143, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_mpg_6_4_1", .symbol = "tf_exl3_mpg_6_4_1", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 128, .static = 128, .bound = 128 }, .{ .regs = 128, .static = 128, .bound = 128 }, .{ .regs = 128, .static = 128, .bound = 128 }, .{ .regs = 128, .static = 128, .bound = 128 }, .{ .regs = 128, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_mpg_6_8_1", .symbol = "tf_exl3_mpg_6_8_1", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 128, .static = 128, .bound = 256 }, .{ .regs = 128, .static = 128, .bound = 256 }, .{ .regs = 128, .static = 128, .bound = 256 }, .{ .regs = 128, .static = 128, .bound = 256 }, .{ .regs = 128, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_mpg_6_2_2", .symbol = "tf_exl3_mpg_6_2_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 238, .static = 128, .bound = 64 }, .{ .regs = 238, .static = 128, .bound = 64 }, .{ .regs = 238, .static = 128, .bound = 64 }, .{ .regs = 230, .static = 128, .bound = 64 }, .{ .regs = 230, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_mpg_6_4_2", .symbol = "tf_exl3_mpg_6_4_2", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 232, .static = 128, .bound = 128 }, .{ .regs = 232, .static = 128, .bound = 128 }, .{ .regs = 232, .static = 128, .bound = 128 }, .{ .regs = 222, .static = 128, .bound = 128 }, .{ .regs = 222, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_mpg_6_8_2", .symbol = "tf_exl3_mpg_6_8_2", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 228, .static = 128, .bound = 256 }, .{ .regs = 232, .static = 128, .bound = 256 }, .{ .regs = 232, .static = 128, .bound = 256 }, .{ .regs = 221, .static = 128, .bound = 256 }, .{ .regs = 221, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_mpg_8_2_1", .symbol = "tf_exl3_mpg_8_2_1", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 128, .static = 128, .bound = 64 }, .{ .regs = 128, .static = 128, .bound = 64 }, .{ .regs = 128, .static = 128, .bound = 64 }, .{ .regs = 126, .static = 128, .bound = 64 }, .{ .regs = 126, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_mpg_8_4_1", .symbol = "tf_exl3_mpg_8_4_1", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 128, .static = 128, .bound = 128 }, .{ .regs = 128, .static = 128, .bound = 128 }, .{ .regs = 128, .static = 128, .bound = 128 }, .{ .regs = 125, .static = 128, .bound = 128 }, .{ .regs = 125, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_mpg_8_8_1", .symbol = "tf_exl3_mpg_8_8_1", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 128, .static = 128, .bound = 256 }, .{ .regs = 128, .static = 128, .bound = 256 }, .{ .regs = 128, .static = 128, .bound = 256 }, .{ .regs = 124, .static = 128, .bound = 256 }, .{ .regs = 124, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_mpg_8_2_2", .symbol = "tf_exl3_mpg_8_2_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 238, .static = 128, .bound = 64 }, .{ .regs = 238, .static = 128, .bound = 64 }, .{ .regs = 238, .static = 128, .bound = 64 }, .{ .regs = 224, .static = 128, .bound = 64 }, .{ .regs = 224, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_mpg_8_4_2", .symbol = "tf_exl3_mpg_8_4_2", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 236, .static = 128, .bound = 128 }, .{ .regs = 236, .static = 128, .bound = 128 }, .{ .regs = 236, .static = 128, .bound = 128 }, .{ .regs = 215, .static = 128, .bound = 128 }, .{ .regs = 215, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_mpg_8_8_2", .symbol = "tf_exl3_mpg_8_8_2", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 239, .static = 128, .bound = 256 }, .{ .regs = 239, .static = 128, .bound = 256 }, .{ .regs = 239, .static = 128, .bound = 256 }, .{ .regs = 220, .static = 128, .bound = 256 }, .{ .regs = 220, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_mpg_10_2_1", .symbol = "tf_exl3_mpg_10_2_1", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 160, .static = 128, .bound = 64 }, .{ .regs = 158, .static = 128, .bound = 64 }, .{ .regs = 158, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 }, .{ .regs = 168, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_mpg_10_4_1", .symbol = "tf_exl3_mpg_10_4_1", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 153, .static = 128, .bound = 128 }, .{ .regs = 155, .static = 128, .bound = 128 }, .{ .regs = 155, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_mpg_10_8_1", .symbol = "tf_exl3_mpg_10_8_1", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 147, .static = 128, .bound = 256 }, .{ .regs = 149, .static = 128, .bound = 256 }, .{ .regs = 149, .static = 128, .bound = 256 }, .{ .regs = 144, .static = 128, .bound = 256 }, .{ .regs = 144, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_mpg_10_2_2", .symbol = "tf_exl3_mpg_10_2_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 241, .static = 128, .bound = 64 }, .{ .regs = 242, .static = 128, .bound = 64 }, .{ .regs = 242, .static = 128, .bound = 64 }, .{ .regs = 255, .static = 128, .bound = 64 }, .{ .regs = 255, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_mpg_10_4_2", .symbol = "tf_exl3_mpg_10_4_2", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 241, .static = 128, .bound = 128 }, .{ .regs = 241, .static = 128, .bound = 128 }, .{ .regs = 241, .static = 128, .bound = 128 }, .{ .regs = 255, .static = 128, .bound = 128 }, .{ .regs = 255, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_mpg_10_8_2", .symbol = "tf_exl3_mpg_10_8_2", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 241, .static = 128, .bound = 256 }, .{ .regs = 241, .static = 128, .bound = 256 }, .{ .regs = 241, .static = 128, .bound = 256 }, .{ .regs = 255, .static = 128, .bound = 256 }, .{ .regs = 255, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_mpg_12_2_1", .symbol = "tf_exl3_mpg_12_2_1", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 160, .static = 128, .bound = 64 }, .{ .regs = 158, .static = 128, .bound = 64 }, .{ .regs = 158, .static = 128, .bound = 64 }, .{ .regs = 160, .static = 128, .bound = 64 }, .{ .regs = 160, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_mpg_12_4_1", .symbol = "tf_exl3_mpg_12_4_1", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 153, .static = 128, .bound = 128 }, .{ .regs = 155, .static = 128, .bound = 128 }, .{ .regs = 155, .static = 128, .bound = 128 }, .{ .regs = 160, .static = 128, .bound = 128 }, .{ .regs = 160, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_mpg_12_8_1", .symbol = "tf_exl3_mpg_12_8_1", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 147, .static = 128, .bound = 256 }, .{ .regs = 149, .static = 128, .bound = 256 }, .{ .regs = 149, .static = 128, .bound = 256 }, .{ .regs = 144, .static = 128, .bound = 256 }, .{ .regs = 144, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_mpg_12_2_2", .symbol = "tf_exl3_mpg_12_2_2", .image = "exl3_linear", .threads = 64, .dynamic = 8192 }, .usage = .{ .{ .regs = 249, .static = 128, .bound = 64 }, .{ .regs = 249, .static = 128, .bound = 64 }, .{ .regs = 249, .static = 128, .bound = 64 }, .{ .regs = 252, .static = 128, .bound = 64 }, .{ .regs = 252, .static = 128, .bound = 64 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_mpg_12_4_2", .symbol = "tf_exl3_mpg_12_4_2", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 249, .static = 128, .bound = 128 }, .{ .regs = 249, .static = 128, .bound = 128 }, .{ .regs = 249, .static = 128, .bound = 128 }, .{ .regs = 252, .static = 128, .bound = 128 }, .{ .regs = 252, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_mpg_12_8_2", .symbol = "tf_exl3_mpg_12_8_2", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 249, .static = 128, .bound = 256 }, .{ .regs = 249, .static = 128, .bound = 256 }, .{ .regs = 249, .static = 128, .bound = 256 }, .{ .regs = 255, .static = 128, .bound = 256 }, .{ .regs = 255, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_wc_8_2_2_0", .symbol = "tf_exl3_wc_8_2_2_0", .image = "exl3_linear", .threads = 256, .dynamic = 16384, .opt_in = true }, .usage = .{ .{ .regs = 98, .static = 128, .bound = 256 }, .{ .regs = 98, .static = 128, .bound = 256 }, .{ .regs = 98, .static = 128, .bound = 256 }, .{ .regs = 60, .static = 128, .bound = 256 }, .{ .regs = 60, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_wc_8_2_4_0", .symbol = "tf_exl3_wc_8_2_4_0", .image = "exl3_linear", .threads = 256, .dynamic = 32768, .opt_in = true }, .usage = .{ .{ .regs = 97, .static = 128, .bound = 256 }, .{ .regs = 96, .static = 128, .bound = 256 }, .{ .regs = 96, .static = 128, .bound = 256 }, .{ .regs = 63, .static = 128, .bound = 256 }, .{ .regs = 63, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_wc_8_4_2_0", .symbol = "tf_exl3_wc_8_4_2_0", .image = "exl3_linear", .threads = 256, .dynamic = 32768, .opt_in = true }, .usage = .{ .{ .regs = 128, .static = 128, .bound = 256 }, .{ .regs = 128, .static = 128, .bound = 256 }, .{ .regs = 128, .static = 128, .bound = 256 }, .{ .regs = 122, .static = 128, .bound = 256 }, .{ .regs = 122, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_wc_8_4_4_0", .symbol = "tf_exl3_wc_8_4_4_0", .image = "exl3_linear", .threads = 256, .dynamic = 49152, .opt_in = true }, .usage = .{ .{ .regs = 128, .static = 128, .bound = 256 }, .{ .regs = 128, .static = 128, .bound = 256 }, .{ .regs = 128, .static = 128, .bound = 256 }, .{ .regs = 125, .static = 128, .bound = 256 }, .{ .regs = 125, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_wc_8_2_2_1", .symbol = "tf_exl3_wc_8_2_2_1", .image = "exl3_linear", .threads = 256, .dynamic = 16384, .opt_in = true }, .usage = .{ .{ .regs = 126, .static = 0, .bound = 256 }, .{ .regs = 126, .static = 0, .bound = 256 }, .{ .regs = 126, .static = 0, .bound = 256 }, .{ .regs = 106, .static = 0, .bound = 256 }, .{ .regs = 106, .static = 0, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_wc_8_2_4_1", .symbol = "tf_exl3_wc_8_2_4_1", .image = "exl3_linear", .threads = 256, .dynamic = 32768, .opt_in = true }, .usage = .{ .{ .regs = 127, .static = 0, .bound = 256 }, .{ .regs = 127, .static = 0, .bound = 256 }, .{ .regs = 127, .static = 0, .bound = 256 }, .{ .regs = 105, .static = 0, .bound = 256 }, .{ .regs = 105, .static = 0, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_wc_8_4_2_1", .symbol = "tf_exl3_wc_8_4_2_1", .image = "exl3_linear", .threads = 256, .dynamic = 32768, .opt_in = true }, .usage = .{ .{ .regs = 206, .static = 0, .bound = 256 }, .{ .regs = 206, .static = 0, .bound = 256 }, .{ .regs = 206, .static = 0, .bound = 256 }, .{ .regs = 179, .static = 0, .bound = 256 }, .{ .regs = 179, .static = 0, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_wc_8_4_4_1", .symbol = "tf_exl3_wc_8_4_4_1", .image = "exl3_linear", .threads = 256, .dynamic = 49152, .opt_in = true }, .usage = .{ .{ .regs = 206, .static = 0, .bound = 256 }, .{ .regs = 206, .static = 0, .bound = 256 }, .{ .regs = 206, .static = 0, .bound = 256 }, .{ .regs = 175, .static = 0, .bound = 256 }, .{ .regs = 175, .static = 0, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_wc_12_2_2_0", .symbol = "tf_exl3_wc_12_2_2_0", .image = "exl3_linear", .threads = 256, .dynamic = 20480, .opt_in = true }, .usage = .{ .{ .regs = 100, .static = 128, .bound = 256 }, .{ .regs = 100, .static = 128, .bound = 256 }, .{ .regs = 100, .static = 128, .bound = 256 }, .{ .regs = 64, .static = 128, .bound = 256 }, .{ .regs = 64, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_wc_12_2_4_0", .symbol = "tf_exl3_wc_12_2_4_0", .image = "exl3_linear", .threads = 256, .dynamic = 40960, .opt_in = true }, .usage = .{ .{ .regs = 101, .static = 128, .bound = 256 }, .{ .regs = 101, .static = 128, .bound = 256 }, .{ .regs = 101, .static = 128, .bound = 256 }, .{ .regs = 60, .static = 128, .bound = 256 }, .{ .regs = 60, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_wc_12_4_2_0", .symbol = "tf_exl3_wc_12_4_2_0", .image = "exl3_linear", .threads = 256, .dynamic = 32768, .opt_in = true }, .usage = .{ .{ .regs = 127, .static = 128, .bound = 256 }, .{ .regs = 127, .static = 128, .bound = 256 }, .{ .regs = 127, .static = 128, .bound = 256 }, .{ .regs = 125, .static = 128, .bound = 256 }, .{ .regs = 125, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_wc_12_4_4_0", .symbol = "tf_exl3_wc_12_4_4_0", .image = "exl3_linear", .threads = 256, .dynamic = 57344, .opt_in = true }, .usage = .{ .{ .regs = 153, .static = 128, .bound = 256 }, .{ .regs = 153, .static = 128, .bound = 256 }, .{ .regs = 153, .static = 128, .bound = 256 }, .{ .regs = 124, .static = 128, .bound = 256 }, .{ .regs = 124, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_wc_12_2_2_1", .symbol = "tf_exl3_wc_12_2_2_1", .image = "exl3_linear", .threads = 256, .dynamic = 20480, .opt_in = true }, .usage = .{ .{ .regs = 127, .static = 0, .bound = 256 }, .{ .regs = 127, .static = 0, .bound = 256 }, .{ .regs = 127, .static = 0, .bound = 256 }, .{ .regs = 107, .static = 0, .bound = 256 }, .{ .regs = 107, .static = 0, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_wc_12_2_4_1", .symbol = "tf_exl3_wc_12_2_4_1", .image = "exl3_linear", .threads = 256, .dynamic = 40960, .opt_in = true }, .usage = .{ .{ .regs = 128, .static = 0, .bound = 256 }, .{ .regs = 128, .static = 0, .bound = 256 }, .{ .regs = 128, .static = 0, .bound = 256 }, .{ .regs = 97, .static = 0, .bound = 256 }, .{ .regs = 97, .static = 0, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_wc_12_4_2_1", .symbol = "tf_exl3_wc_12_4_2_1", .image = "exl3_linear", .threads = 256, .dynamic = 32768, .opt_in = true }, .usage = .{ .{ .regs = 216, .static = 0, .bound = 256 }, .{ .regs = 216, .static = 0, .bound = 256 }, .{ .regs = 216, .static = 0, .bound = 256 }, .{ .regs = 178, .static = 0, .bound = 256 }, .{ .regs = 178, .static = 0, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_wc_12_4_4_1", .symbol = "tf_exl3_wc_12_4_4_1", .image = "exl3_linear", .threads = 256, .dynamic = 57344, .opt_in = true }, .usage = .{ .{ .regs = 215, .static = 0, .bound = 256 }, .{ .regs = 215, .static = 0, .bound = 256 }, .{ .regs = 215, .static = 0, .bound = 256 }, .{ .regs = 182, .static = 0, .bound = 256 }, .{ .regs = 182, .static = 0, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_group_8_2_4", .symbol = "tf_exl3_group_8_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 201, .static = 128, .bound = 128 }, .{ .regs = 201, .static = 128, .bound = 128 }, .{ .regs = 201, .static = 128, .bound = 128 }, .{ .regs = 163, .static = 128, .bound = 128 }, .{ .regs = 163, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_group_8_2_8", .symbol = "tf_exl3_group_8_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 164, .static = 128, .bound = 256 }, .{ .regs = 162, .static = 128, .bound = 256 }, .{ .regs = 162, .static = 128, .bound = 256 }, .{ .regs = 128, .static = 128, .bound = 256 }, .{ .regs = 128, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_rot_group", .symbol = "tf_exl3_rot_group", .image = "exl3_linear", .threads = 128 }, .usage = .{ .{ .regs = 24, .static = 0, .bound = 128 }, .{ .regs = 22, .static = 0, .bound = 128 }, .{ .regs = 22, .static = 0, .bound = 128 }, .{ .regs = 24, .static = 0, .bound = 128 }, .{ .regs = 24, .static = 0, .bound = 128 } } },
+};
diff --git a/zig/src/core/contract/registry.zig b/zig/src/core/contract/registry.zig
index 64b8cbd84b..38bc50be6f 100644
--- a/zig/src/core/contract/registry.zig
+++ b/zig/src/core/contract/registry.zig
@@ -115,66 +115,66 @@ pub const cuda = @import("flashnext_cuda_records.zig").cuda ++ [_]CudaRecord{
     .{ .launch = .{ .name = "flashnext_attn fn_qsa_pool_media", .symbol = "fn_qsa_pool_media", .image = "flashnext_attn", .threads = 128 }, .usage = .{ .{ .regs = 32, .static = 1152, .bound = 128 }, .{ .regs = 40, .static = 1152, .bound = 128 }, .{ .regs = 40, .static = 1152, .bound = 128 }, .{ .regs = 39, .static = 1152, .bound = 128 }, .{ .regs = 39, .static = 1152, .bound = 128 } } },
     // EXL3: the linear's instantiations at their most rows' shared bytes, the routed experts, Dense's tile candidate
     .{ .launch = .{ .name = "exl3_linear tf_exl3_rot_in", .symbol = "tf_exl3_rot_in", .image = "exl3_linear", .threads = 128 }, .usage = .{ .{ .regs = 19, .static = 0, .bound = 128 }, .{ .regs = 22, .static = 0, .bound = 128 }, .{ .regs = 22, .static = 0, .bound = 128 }, .{ .regs = 22, .static = 0, .bound = 128 }, .{ .regs = 22, .static = 0, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_2_2_4", .symbol = "tf_exl3_linear_2_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 152, .static = 16, .bound = 128 }, .{ .regs = 152, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_4_2_4", .symbol = "tf_exl3_linear_4_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 152, .static = 16, .bound = 128 }, .{ .regs = 152, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_6_2_4", .symbol = "tf_exl3_linear_6_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 164, .static = 16, .bound = 128 }, .{ .regs = 164, .static = 16, .bound = 128 }, .{ .regs = 164, .static = 16, .bound = 128 }, .{ .regs = 155, .static = 16, .bound = 128 }, .{ .regs = 155, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_8_2_4", .symbol = "tf_exl3_linear_8_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 154, .static = 16, .bound = 128 }, .{ .regs = 154, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_10_2_4", .symbol = "tf_exl3_linear_10_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_12_2_4", .symbol = "tf_exl3_linear_12_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_14_2_4", .symbol = "tf_exl3_linear_14_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 167, .static = 16, .bound = 128 }, .{ .regs = 167, .static = 16, .bound = 128 }, .{ .regs = 128, .static = 16, .bound = 128 }, .{ .regs = 128, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_16_2_4", .symbol = "tf_exl3_linear_16_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 128, .static = 16, .bound = 128 }, .{ .regs = 128, .static = 16, .bound = 128 }, .{ .regs = 128, .static = 16, .bound = 128 }, .{ .regs = 125, .static = 16, .bound = 128 }, .{ .regs = 125, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_3_2_4", .symbol = "tf_exl3_linear_3_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 164, .static = 16, .bound = 128 }, .{ .regs = 164, .static = 16, .bound = 128 }, .{ .regs = 164, .static = 16, .bound = 128 }, .{ .regs = 153, .static = 16, .bound = 128 }, .{ .regs = 153, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_5_2_4", .symbol = "tf_exl3_linear_5_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 164, .static = 16, .bound = 128 }, .{ .regs = 164, .static = 16, .bound = 128 }, .{ .regs = 164, .static = 16, .bound = 128 }, .{ .regs = 155, .static = 16, .bound = 128 }, .{ .regs = 155, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_7_2_4", .symbol = "tf_exl3_linear_7_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_9_2_4", .symbol = "tf_exl3_linear_9_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_11_2_4", .symbol = "tf_exl3_linear_11_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_13_2_4", .symbol = "tf_exl3_linear_13_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 167, .static = 16, .bound = 128 }, .{ .regs = 167, .static = 16, .bound = 128 }, .{ .regs = 126, .static = 16, .bound = 128 }, .{ .regs = 126, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_2_1_4", .symbol = "tf_exl3_linear_2_1_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 153, .static = 16, .bound = 128 }, .{ .regs = 153, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_4_1_4", .symbol = "tf_exl3_linear_4_1_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 153, .static = 16, .bound = 128 }, .{ .regs = 153, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_6_1_4", .symbol = "tf_exl3_linear_6_1_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 164, .static = 16, .bound = 128 }, .{ .regs = 164, .static = 16, .bound = 128 }, .{ .regs = 164, .static = 16, .bound = 128 }, .{ .regs = 154, .static = 16, .bound = 128 }, .{ .regs = 154, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_8_1_4", .symbol = "tf_exl3_linear_8_1_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 150, .static = 16, .bound = 128 }, .{ .regs = 150, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_10_1_4", .symbol = "tf_exl3_linear_10_1_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_12_1_4", .symbol = "tf_exl3_linear_12_1_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_14_1_4", .symbol = "tf_exl3_linear_14_1_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 166, .static = 16, .bound = 128 }, .{ .regs = 166, .static = 16, .bound = 128 }, .{ .regs = 128, .static = 16, .bound = 128 }, .{ .regs = 128, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_16_1_4", .symbol = "tf_exl3_linear_16_1_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 128, .static = 16, .bound = 128 }, .{ .regs = 128, .static = 16, .bound = 128 }, .{ .regs = 128, .static = 16, .bound = 128 }, .{ .regs = 123, .static = 16, .bound = 128 }, .{ .regs = 123, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_2_0_4", .symbol = "tf_exl3_linear_2_0_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 151, .static = 16, .bound = 128 }, .{ .regs = 151, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_4_0_4", .symbol = "tf_exl3_linear_4_0_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 151, .static = 16, .bound = 128 }, .{ .regs = 151, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_6_0_4", .symbol = "tf_exl3_linear_6_0_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 164, .static = 16, .bound = 128 }, .{ .regs = 164, .static = 16, .bound = 128 }, .{ .regs = 164, .static = 16, .bound = 128 }, .{ .regs = 158, .static = 16, .bound = 128 }, .{ .regs = 158, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_8_0_4", .symbol = "tf_exl3_linear_8_0_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 159, .static = 16, .bound = 128 }, .{ .regs = 159, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_10_0_4", .symbol = "tf_exl3_linear_10_0_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_12_0_4", .symbol = "tf_exl3_linear_12_0_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_14_0_4", .symbol = "tf_exl3_linear_14_0_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 168, .static = 16, .bound = 128 }, .{ .regs = 126, .static = 16, .bound = 128 }, .{ .regs = 126, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_16_0_4", .symbol = "tf_exl3_linear_16_0_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 128, .static = 16, .bound = 128 }, .{ .regs = 128, .static = 16, .bound = 128 }, .{ .regs = 128, .static = 16, .bound = 128 }, .{ .regs = 126, .static = 16, .bound = 128 }, .{ .regs = 126, .static = 16, .bound = 128 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_2_2_8", .symbol = "tf_exl3_linear_2_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 196, .static = 16, .bound = 256 }, .{ .regs = 196, .static = 16, .bound = 256 }, .{ .regs = 196, .static = 16, .bound = 256 }, .{ .regs = 126, .static = 16, .bound = 256 }, .{ .regs = 126, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_4_2_8", .symbol = "tf_exl3_linear_4_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 196, .static = 16, .bound = 256 }, .{ .regs = 196, .static = 16, .bound = 256 }, .{ .regs = 196, .static = 16, .bound = 256 }, .{ .regs = 126, .static = 16, .bound = 256 }, .{ .regs = 126, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_6_2_8", .symbol = "tf_exl3_linear_6_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 159, .static = 16, .bound = 256 }, .{ .regs = 161, .static = 16, .bound = 256 }, .{ .regs = 161, .static = 16, .bound = 256 }, .{ .regs = 128, .static = 16, .bound = 256 }, .{ .regs = 128, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_8_2_8", .symbol = "tf_exl3_linear_8_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 202, .static = 16, .bound = 256 }, .{ .regs = 202, .static = 16, .bound = 256 }, .{ .regs = 202, .static = 16, .bound = 256 }, .{ .regs = 126, .static = 16, .bound = 256 }, .{ .regs = 126, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_10_2_8", .symbol = "tf_exl3_linear_10_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 178, .static = 16, .bound = 256 }, .{ .regs = 178, .static = 16, .bound = 256 }, .{ .regs = 178, .static = 16, .bound = 256 }, .{ .regs = 223, .static = 16, .bound = 256 }, .{ .regs = 223, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_12_2_8", .symbol = "tf_exl3_linear_12_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 181, .static = 16, .bound = 256 }, .{ .regs = 181, .static = 16, .bound = 256 }, .{ .regs = 181, .static = 16, .bound = 256 }, .{ .regs = 222, .static = 16, .bound = 256 }, .{ .regs = 222, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_14_2_8", .symbol = "tf_exl3_linear_14_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 225, .static = 16, .bound = 256 }, .{ .regs = 225, .static = 16, .bound = 256 }, .{ .regs = 225, .static = 16, .bound = 256 }, .{ .regs = 127, .static = 16, .bound = 256 }, .{ .regs = 127, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_16_2_8", .symbol = "tf_exl3_linear_16_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 214, .static = 16, .bound = 256 }, .{ .regs = 214, .static = 16, .bound = 256 }, .{ .regs = 214, .static = 16, .bound = 256 }, .{ .regs = 124, .static = 16, .bound = 256 }, .{ .regs = 124, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_3_2_8", .symbol = "tf_exl3_linear_3_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 159, .static = 16, .bound = 256 }, .{ .regs = 161, .static = 16, .bound = 256 }, .{ .regs = 161, .static = 16, .bound = 256 }, .{ .regs = 128, .static = 16, .bound = 256 }, .{ .regs = 128, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_5_2_8", .symbol = "tf_exl3_linear_5_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 160, .static = 16, .bound = 256 }, .{ .regs = 161, .static = 16, .bound = 256 }, .{ .regs = 161, .static = 16, .bound = 256 }, .{ .regs = 128, .static = 16, .bound = 256 }, .{ .regs = 128, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_7_2_8", .symbol = "tf_exl3_linear_7_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 179, .static = 16, .bound = 256 }, .{ .regs = 179, .static = 16, .bound = 256 }, .{ .regs = 179, .static = 16, .bound = 256 }, .{ .regs = 232, .static = 16, .bound = 256 }, .{ .regs = 232, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_9_2_8", .symbol = "tf_exl3_linear_9_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 176, .static = 16, .bound = 256 }, .{ .regs = 177, .static = 16, .bound = 256 }, .{ .regs = 177, .static = 16, .bound = 256 }, .{ .regs = 220, .static = 16, .bound = 256 }, .{ .regs = 220, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_11_2_8", .symbol = "tf_exl3_linear_11_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 181, .static = 16, .bound = 256 }, .{ .regs = 181, .static = 16, .bound = 256 }, .{ .regs = 181, .static = 16, .bound = 256 }, .{ .regs = 222, .static = 16, .bound = 256 }, .{ .regs = 222, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_13_2_8", .symbol = "tf_exl3_linear_13_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 216, .static = 16, .bound = 256 }, .{ .regs = 216, .static = 16, .bound = 256 }, .{ .regs = 216, .static = 16, .bound = 256 }, .{ .regs = 127, .static = 16, .bound = 256 }, .{ .regs = 127, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_2_1_8", .symbol = "tf_exl3_linear_2_1_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 224, .static = 16, .bound = 256 }, .{ .regs = 224, .static = 16, .bound = 256 }, .{ .regs = 224, .static = 16, .bound = 256 }, .{ .regs = 126, .static = 16, .bound = 256 }, .{ .regs = 126, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_4_1_8", .symbol = "tf_exl3_linear_4_1_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 224, .static = 16, .bound = 256 }, .{ .regs = 224, .static = 16, .bound = 256 }, .{ .regs = 224, .static = 16, .bound = 256 }, .{ .regs = 126, .static = 16, .bound = 256 }, .{ .regs = 126, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_6_1_8", .symbol = "tf_exl3_linear_6_1_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 160, .static = 16, .bound = 256 }, .{ .regs = 161, .static = 16, .bound = 256 }, .{ .regs = 161, .static = 16, .bound = 256 }, .{ .regs = 128, .static = 16, .bound = 256 }, .{ .regs = 128, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_8_1_8", .symbol = "tf_exl3_linear_8_1_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 225, .static = 16, .bound = 256 }, .{ .regs = 225, .static = 16, .bound = 256 }, .{ .regs = 225, .static = 16, .bound = 256 }, .{ .regs = 126, .static = 16, .bound = 256 }, .{ .regs = 126, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_10_1_8", .symbol = "tf_exl3_linear_10_1_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 203, .static = 16, .bound = 256 }, .{ .regs = 203, .static = 16, .bound = 256 }, .{ .regs = 203, .static = 16, .bound = 256 }, .{ .regs = 224, .static = 16, .bound = 256 }, .{ .regs = 224, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_12_1_8", .symbol = "tf_exl3_linear_12_1_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 200, .static = 16, .bound = 256 }, .{ .regs = 200, .static = 16, .bound = 256 }, .{ .regs = 200, .static = 16, .bound = 256 }, .{ .regs = 220, .static = 16, .bound = 256 }, .{ .regs = 220, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_14_1_8", .symbol = "tf_exl3_linear_14_1_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 227, .static = 16, .bound = 256 }, .{ .regs = 227, .static = 16, .bound = 256 }, .{ .regs = 227, .static = 16, .bound = 256 }, .{ .regs = 128, .static = 16, .bound = 256 }, .{ .regs = 128, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_16_1_8", .symbol = "tf_exl3_linear_16_1_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 218, .static = 16, .bound = 256 }, .{ .regs = 218, .static = 16, .bound = 256 }, .{ .regs = 218, .static = 16, .bound = 256 }, .{ .regs = 124, .static = 16, .bound = 256 }, .{ .regs = 124, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_2_0_8", .symbol = "tf_exl3_linear_2_0_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 222, .static = 16, .bound = 256 }, .{ .regs = 222, .static = 16, .bound = 256 }, .{ .regs = 222, .static = 16, .bound = 256 }, .{ .regs = 126, .static = 16, .bound = 256 }, .{ .regs = 126, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_4_0_8", .symbol = "tf_exl3_linear_4_0_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 222, .static = 16, .bound = 256 }, .{ .regs = 222, .static = 16, .bound = 256 }, .{ .regs = 222, .static = 16, .bound = 256 }, .{ .regs = 126, .static = 16, .bound = 256 }, .{ .regs = 126, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_6_0_8", .symbol = "tf_exl3_linear_6_0_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 159, .static = 16, .bound = 256 }, .{ .regs = 161, .static = 16, .bound = 256 }, .{ .regs = 161, .static = 16, .bound = 256 }, .{ .regs = 128, .static = 16, .bound = 256 }, .{ .regs = 128, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_8_0_8", .symbol = "tf_exl3_linear_8_0_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 228, .static = 16, .bound = 256 }, .{ .regs = 228, .static = 16, .bound = 256 }, .{ .regs = 228, .static = 16, .bound = 256 }, .{ .regs = 126, .static = 16, .bound = 256 }, .{ .regs = 126, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_10_0_8", .symbol = "tf_exl3_linear_10_0_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 196, .static = 16, .bound = 256 }, .{ .regs = 196, .static = 16, .bound = 256 }, .{ .regs = 196, .static = 16, .bound = 256 }, .{ .regs = 223, .static = 16, .bound = 256 }, .{ .regs = 223, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_12_0_8", .symbol = "tf_exl3_linear_12_0_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 198, .static = 16, .bound = 256 }, .{ .regs = 198, .static = 16, .bound = 256 }, .{ .regs = 198, .static = 16, .bound = 256 }, .{ .regs = 226, .static = 16, .bound = 256 }, .{ .regs = 226, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_14_0_8", .symbol = "tf_exl3_linear_14_0_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 225, .static = 16, .bound = 256 }, .{ .regs = 225, .static = 16, .bound = 256 }, .{ .regs = 225, .static = 16, .bound = 256 }, .{ .regs = 127, .static = 16, .bound = 256 }, .{ .regs = 127, .static = 16, .bound = 256 } } },
-    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_16_0_8", .symbol = "tf_exl3_linear_16_0_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 220, .static = 16, .bound = 256 }, .{ .regs = 220, .static = 16, .bound = 256 }, .{ .regs = 220, .static = 16, .bound = 256 }, .{ .regs = 125, .static = 16, .bound = 256 }, .{ .regs = 125, .static = 16, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_2_2_4", .symbol = "tf_exl3_linear_2_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 152, .static = 128, .bound = 128 }, .{ .regs = 152, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_4_2_4", .symbol = "tf_exl3_linear_4_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 152, .static = 128, .bound = 128 }, .{ .regs = 152, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_6_2_4", .symbol = "tf_exl3_linear_6_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 164, .static = 128, .bound = 128 }, .{ .regs = 164, .static = 128, .bound = 128 }, .{ .regs = 164, .static = 128, .bound = 128 }, .{ .regs = 155, .static = 128, .bound = 128 }, .{ .regs = 155, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_8_2_4", .symbol = "tf_exl3_linear_8_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 154, .static = 128, .bound = 128 }, .{ .regs = 154, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_10_2_4", .symbol = "tf_exl3_linear_10_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_12_2_4", .symbol = "tf_exl3_linear_12_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_14_2_4", .symbol = "tf_exl3_linear_14_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 167, .static = 128, .bound = 128 }, .{ .regs = 167, .static = 128, .bound = 128 }, .{ .regs = 128, .static = 128, .bound = 128 }, .{ .regs = 128, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_16_2_4", .symbol = "tf_exl3_linear_16_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 128, .static = 128, .bound = 128 }, .{ .regs = 128, .static = 128, .bound = 128 }, .{ .regs = 128, .static = 128, .bound = 128 }, .{ .regs = 125, .static = 128, .bound = 128 }, .{ .regs = 125, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_3_2_4", .symbol = "tf_exl3_linear_3_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 164, .static = 128, .bound = 128 }, .{ .regs = 164, .static = 128, .bound = 128 }, .{ .regs = 164, .static = 128, .bound = 128 }, .{ .regs = 153, .static = 128, .bound = 128 }, .{ .regs = 153, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_5_2_4", .symbol = "tf_exl3_linear_5_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 164, .static = 128, .bound = 128 }, .{ .regs = 164, .static = 128, .bound = 128 }, .{ .regs = 164, .static = 128, .bound = 128 }, .{ .regs = 155, .static = 128, .bound = 128 }, .{ .regs = 155, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_7_2_4", .symbol = "tf_exl3_linear_7_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_9_2_4", .symbol = "tf_exl3_linear_9_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_11_2_4", .symbol = "tf_exl3_linear_11_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_13_2_4", .symbol = "tf_exl3_linear_13_2_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 167, .static = 128, .bound = 128 }, .{ .regs = 167, .static = 128, .bound = 128 }, .{ .regs = 126, .static = 128, .bound = 128 }, .{ .regs = 126, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_2_1_4", .symbol = "tf_exl3_linear_2_1_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 153, .static = 128, .bound = 128 }, .{ .regs = 153, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_4_1_4", .symbol = "tf_exl3_linear_4_1_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 153, .static = 128, .bound = 128 }, .{ .regs = 153, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_6_1_4", .symbol = "tf_exl3_linear_6_1_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 164, .static = 128, .bound = 128 }, .{ .regs = 164, .static = 128, .bound = 128 }, .{ .regs = 164, .static = 128, .bound = 128 }, .{ .regs = 154, .static = 128, .bound = 128 }, .{ .regs = 154, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_8_1_4", .symbol = "tf_exl3_linear_8_1_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 150, .static = 128, .bound = 128 }, .{ .regs = 150, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_10_1_4", .symbol = "tf_exl3_linear_10_1_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_12_1_4", .symbol = "tf_exl3_linear_12_1_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_14_1_4", .symbol = "tf_exl3_linear_14_1_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 166, .static = 128, .bound = 128 }, .{ .regs = 166, .static = 128, .bound = 128 }, .{ .regs = 128, .static = 128, .bound = 128 }, .{ .regs = 128, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_16_1_4", .symbol = "tf_exl3_linear_16_1_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 128, .static = 128, .bound = 128 }, .{ .regs = 128, .static = 128, .bound = 128 }, .{ .regs = 128, .static = 128, .bound = 128 }, .{ .regs = 123, .static = 128, .bound = 128 }, .{ .regs = 123, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_2_0_4", .symbol = "tf_exl3_linear_2_0_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 151, .static = 128, .bound = 128 }, .{ .regs = 151, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_4_0_4", .symbol = "tf_exl3_linear_4_0_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 151, .static = 128, .bound = 128 }, .{ .regs = 151, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_6_0_4", .symbol = "tf_exl3_linear_6_0_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 164, .static = 128, .bound = 128 }, .{ .regs = 164, .static = 128, .bound = 128 }, .{ .regs = 164, .static = 128, .bound = 128 }, .{ .regs = 158, .static = 128, .bound = 128 }, .{ .regs = 158, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_8_0_4", .symbol = "tf_exl3_linear_8_0_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 159, .static = 128, .bound = 128 }, .{ .regs = 159, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_10_0_4", .symbol = "tf_exl3_linear_10_0_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_12_0_4", .symbol = "tf_exl3_linear_12_0_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_14_0_4", .symbol = "tf_exl3_linear_14_0_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 168, .static = 128, .bound = 128 }, .{ .regs = 126, .static = 128, .bound = 128 }, .{ .regs = 126, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_16_0_4", .symbol = "tf_exl3_linear_16_0_4", .image = "exl3_linear", .threads = 128, .dynamic = 16384 }, .usage = .{ .{ .regs = 128, .static = 128, .bound = 128 }, .{ .regs = 128, .static = 128, .bound = 128 }, .{ .regs = 128, .static = 128, .bound = 128 }, .{ .regs = 126, .static = 128, .bound = 128 }, .{ .regs = 126, .static = 128, .bound = 128 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_2_2_8", .symbol = "tf_exl3_linear_2_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 196, .static = 128, .bound = 256 }, .{ .regs = 196, .static = 128, .bound = 256 }, .{ .regs = 196, .static = 128, .bound = 256 }, .{ .regs = 126, .static = 128, .bound = 256 }, .{ .regs = 126, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_4_2_8", .symbol = "tf_exl3_linear_4_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 196, .static = 128, .bound = 256 }, .{ .regs = 196, .static = 128, .bound = 256 }, .{ .regs = 196, .static = 128, .bound = 256 }, .{ .regs = 126, .static = 128, .bound = 256 }, .{ .regs = 126, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_6_2_8", .symbol = "tf_exl3_linear_6_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 159, .static = 128, .bound = 256 }, .{ .regs = 161, .static = 128, .bound = 256 }, .{ .regs = 161, .static = 128, .bound = 256 }, .{ .regs = 128, .static = 128, .bound = 256 }, .{ .regs = 128, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_8_2_8", .symbol = "tf_exl3_linear_8_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 202, .static = 128, .bound = 256 }, .{ .regs = 202, .static = 128, .bound = 256 }, .{ .regs = 202, .static = 128, .bound = 256 }, .{ .regs = 126, .static = 128, .bound = 256 }, .{ .regs = 126, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_10_2_8", .symbol = "tf_exl3_linear_10_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 178, .static = 128, .bound = 256 }, .{ .regs = 178, .static = 128, .bound = 256 }, .{ .regs = 178, .static = 128, .bound = 256 }, .{ .regs = 223, .static = 128, .bound = 256 }, .{ .regs = 223, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_12_2_8", .symbol = "tf_exl3_linear_12_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 181, .static = 128, .bound = 256 }, .{ .regs = 181, .static = 128, .bound = 256 }, .{ .regs = 181, .static = 128, .bound = 256 }, .{ .regs = 222, .static = 128, .bound = 256 }, .{ .regs = 222, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_14_2_8", .symbol = "tf_exl3_linear_14_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 225, .static = 128, .bound = 256 }, .{ .regs = 225, .static = 128, .bound = 256 }, .{ .regs = 225, .static = 128, .bound = 256 }, .{ .regs = 127, .static = 128, .bound = 256 }, .{ .regs = 127, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_16_2_8", .symbol = "tf_exl3_linear_16_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 214, .static = 128, .bound = 256 }, .{ .regs = 214, .static = 128, .bound = 256 }, .{ .regs = 214, .static = 128, .bound = 256 }, .{ .regs = 124, .static = 128, .bound = 256 }, .{ .regs = 124, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_3_2_8", .symbol = "tf_exl3_linear_3_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 159, .static = 128, .bound = 256 }, .{ .regs = 161, .static = 128, .bound = 256 }, .{ .regs = 161, .static = 128, .bound = 256 }, .{ .regs = 128, .static = 128, .bound = 256 }, .{ .regs = 128, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_5_2_8", .symbol = "tf_exl3_linear_5_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 160, .static = 128, .bound = 256 }, .{ .regs = 161, .static = 128, .bound = 256 }, .{ .regs = 161, .static = 128, .bound = 256 }, .{ .regs = 128, .static = 128, .bound = 256 }, .{ .regs = 128, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_7_2_8", .symbol = "tf_exl3_linear_7_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 179, .static = 128, .bound = 256 }, .{ .regs = 179, .static = 128, .bound = 256 }, .{ .regs = 179, .static = 128, .bound = 256 }, .{ .regs = 232, .static = 128, .bound = 256 }, .{ .regs = 232, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_9_2_8", .symbol = "tf_exl3_linear_9_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 176, .static = 128, .bound = 256 }, .{ .regs = 177, .static = 128, .bound = 256 }, .{ .regs = 177, .static = 128, .bound = 256 }, .{ .regs = 220, .static = 128, .bound = 256 }, .{ .regs = 220, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_11_2_8", .symbol = "tf_exl3_linear_11_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 181, .static = 128, .bound = 256 }, .{ .regs = 181, .static = 128, .bound = 256 }, .{ .regs = 181, .static = 128, .bound = 256 }, .{ .regs = 222, .static = 128, .bound = 256 }, .{ .regs = 222, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_13_2_8", .symbol = "tf_exl3_linear_13_2_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 216, .static = 128, .bound = 256 }, .{ .regs = 216, .static = 128, .bound = 256 }, .{ .regs = 216, .static = 128, .bound = 256 }, .{ .regs = 127, .static = 128, .bound = 256 }, .{ .regs = 127, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_2_1_8", .symbol = "tf_exl3_linear_2_1_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 224, .static = 128, .bound = 256 }, .{ .regs = 224, .static = 128, .bound = 256 }, .{ .regs = 224, .static = 128, .bound = 256 }, .{ .regs = 126, .static = 128, .bound = 256 }, .{ .regs = 126, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_4_1_8", .symbol = "tf_exl3_linear_4_1_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 224, .static = 128, .bound = 256 }, .{ .regs = 224, .static = 128, .bound = 256 }, .{ .regs = 224, .static = 128, .bound = 256 }, .{ .regs = 126, .static = 128, .bound = 256 }, .{ .regs = 126, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_6_1_8", .symbol = "tf_exl3_linear_6_1_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 160, .static = 128, .bound = 256 }, .{ .regs = 161, .static = 128, .bound = 256 }, .{ .regs = 161, .static = 128, .bound = 256 }, .{ .regs = 128, .static = 128, .bound = 256 }, .{ .regs = 128, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_8_1_8", .symbol = "tf_exl3_linear_8_1_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 225, .static = 128, .bound = 256 }, .{ .regs = 225, .static = 128, .bound = 256 }, .{ .regs = 225, .static = 128, .bound = 256 }, .{ .regs = 126, .static = 128, .bound = 256 }, .{ .regs = 126, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_10_1_8", .symbol = "tf_exl3_linear_10_1_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 203, .static = 128, .bound = 256 }, .{ .regs = 203, .static = 128, .bound = 256 }, .{ .regs = 203, .static = 128, .bound = 256 }, .{ .regs = 224, .static = 128, .bound = 256 }, .{ .regs = 224, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_12_1_8", .symbol = "tf_exl3_linear_12_1_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 200, .static = 128, .bound = 256 }, .{ .regs = 200, .static = 128, .bound = 256 }, .{ .regs = 200, .static = 128, .bound = 256 }, .{ .regs = 220, .static = 128, .bound = 256 }, .{ .regs = 220, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_14_1_8", .symbol = "tf_exl3_linear_14_1_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 227, .static = 128, .bound = 256 }, .{ .regs = 227, .static = 128, .bound = 256 }, .{ .regs = 227, .static = 128, .bound = 256 }, .{ .regs = 128, .static = 128, .bound = 256 }, .{ .regs = 128, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_16_1_8", .symbol = "tf_exl3_linear_16_1_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 218, .static = 128, .bound = 256 }, .{ .regs = 218, .static = 128, .bound = 256 }, .{ .regs = 218, .static = 128, .bound = 256 }, .{ .regs = 122, .static = 128, .bound = 256 }, .{ .regs = 122, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_2_0_8", .symbol = "tf_exl3_linear_2_0_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 222, .static = 128, .bound = 256 }, .{ .regs = 222, .static = 128, .bound = 256 }, .{ .regs = 222, .static = 128, .bound = 256 }, .{ .regs = 126, .static = 128, .bound = 256 }, .{ .regs = 126, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_4_0_8", .symbol = "tf_exl3_linear_4_0_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 222, .static = 128, .bound = 256 }, .{ .regs = 222, .static = 128, .bound = 256 }, .{ .regs = 222, .static = 128, .bound = 256 }, .{ .regs = 126, .static = 128, .bound = 256 }, .{ .regs = 126, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_6_0_8", .symbol = "tf_exl3_linear_6_0_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 159, .static = 128, .bound = 256 }, .{ .regs = 161, .static = 128, .bound = 256 }, .{ .regs = 161, .static = 128, .bound = 256 }, .{ .regs = 128, .static = 128, .bound = 256 }, .{ .regs = 128, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_8_0_8", .symbol = "tf_exl3_linear_8_0_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 228, .static = 128, .bound = 256 }, .{ .regs = 228, .static = 128, .bound = 256 }, .{ .regs = 228, .static = 128, .bound = 256 }, .{ .regs = 126, .static = 128, .bound = 256 }, .{ .regs = 126, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_10_0_8", .symbol = "tf_exl3_linear_10_0_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 196, .static = 128, .bound = 256 }, .{ .regs = 196, .static = 128, .bound = 256 }, .{ .regs = 196, .static = 128, .bound = 256 }, .{ .regs = 223, .static = 128, .bound = 256 }, .{ .regs = 223, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_12_0_8", .symbol = "tf_exl3_linear_12_0_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 198, .static = 128, .bound = 256 }, .{ .regs = 198, .static = 128, .bound = 256 }, .{ .regs = 198, .static = 128, .bound = 256 }, .{ .regs = 226, .static = 128, .bound = 256 }, .{ .regs = 226, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_14_0_8", .symbol = "tf_exl3_linear_14_0_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 225, .static = 128, .bound = 256 }, .{ .regs = 225, .static = 128, .bound = 256 }, .{ .regs = 225, .static = 128, .bound = 256 }, .{ .regs = 127, .static = 128, .bound = 256 }, .{ .regs = 127, .static = 128, .bound = 256 } } },
+    .{ .launch = .{ .name = "exl3_linear tf_exl3_linear_16_0_8", .symbol = "tf_exl3_linear_16_0_8", .image = "exl3_linear", .threads = 256, .dynamic = 32768 }, .usage = .{ .{ .regs = 220, .static = 128, .bound = 256 }, .{ .regs = 220, .static = 128, .bound = 256 }, .{ .regs = 220, .static = 128, .bound = 256 }, .{ .regs = 125, .static = 128, .bound = 256 }, .{ .regs = 125, .static = 128, .bound = 256 } } },
     .{ .launch = .{ .name = "exl3_experts tf_exl3_group", .symbol = "tf_exl3_group", .image = "exl3_experts", .threads = 1024, .dynamic = 49152, .opt_in = true }, .usage = .{ .{ .regs = 32, .static = 128, .bound = 1024 }, .{ .regs = 42, .static = 128, .bound = 1024 }, .{ .regs = 42, .static = 128, .bound = 1024 }, .{ .regs = 42, .static = 128, .bound = 1024 }, .{ .regs = 42, .static = 128, .bound = 1024 } } },
     .{ .launch = .{ .name = "exl3_experts tf_exl3_experts_rot_in", .symbol = "tf_exl3_experts_rot_in", .image = "exl3_experts", .threads = 32 }, .usage = .{ .{ .regs = 24, .static = 0, .bound = 0 }, .{ .regs = 24, .static = 0, .bound = 0 }, .{ .regs = 24, .static = 0, .bound = 0 }, .{ .regs = 19, .static = 0, .bound = 0 }, .{ .regs = 19, .static = 0, .bound = 0 } } },
     .{ .launch = .{ .name = "exl3_experts tf_exl3_grouped_0", .symbol = "tf_exl3_grouped_0", .image = "exl3_experts", .threads = 128 }, .usage = .{ .{ .regs = 128, .static = 32832, .bound = 0 }, .{ .regs = 128, .static = 32832, .bound = 0 }, .{ .regs = 128, .static = 32832, .bound = 0 }, .{ .regs = 127, .static = 32832, .bound = 0 }, .{ .regs = 127, .static = 32832, .bound = 0 } } },
@@ -205,7 +205,7 @@ pub const cuda = @import("flashnext_cuda_records.zig").cuda ++ [_]CudaRecord{
     .{ .launch = .{ .name = "experts3_prefill down", .symbol = "_ZN19tf_experts3_prefill14prefill_kernelILi1ELi3ELi2ELi2ELi4EEEvPK13__nv_bfloat16iiPK5uint4iiPKiS8_S8_PvifS6_i", .image = "experts3_prefill", .threads = 256, .dynamic = 47104, .opt_in = true }, .usage = .{ .{ .regs = 128, .static = 0, .bound = 256 }, .{ .regs = 128, .static = 0, .bound = 256 }, .{ .regs = 128, .static = 0, .bound = 256 }, .{ .regs = 98, .static = 0, .bound = 256 }, .{ .regs = 98, .static = 0, .bound = 256 } } },
     .{ .launch = .{ .name = "experts3 tf_experts3_widen", .symbol = "tf_experts3_widen", .image = "experts3", .threads = 256 }, .usage = .{ .{ .regs = 32, .static = 0, .bound = 0 }, .{ .regs = 36, .static = 0, .bound = 0 }, .{ .regs = 36, .static = 0, .bound = 0 }, .{ .regs = 32, .static = 0, .bound = 0 }, .{ .regs = 32, .static = 0, .bound = 0 } } },
     .{ .launch = .{ .name = "experts3 tf_experts3_planes", .symbol = "tf_experts3_planes", .image = "experts3", .threads = 256 }, .usage = .{ .{ .regs = 26, .static = 0, .bound = 0 }, .{ .regs = 28, .static = 0, .bound = 0 }, .{ .regs = 28, .static = 0, .bound = 0 }, .{ .regs = 36, .static = 0, .bound = 0 }, .{ .regs = 36, .static = 0, .bound = 0 } } },
-} ++ @import("registry_media.zig").cuda ++ @import("nemotron_prompt_records.zig").cuda ++ @import("glm_cuda_records.zig").cuda ++ @import("registry_shared_attention.zig").records(CudaRecord) ++ @import("gemma_cuda_records.zig").cuda ++ @import("spark_cuda_records.zig").cuda ++ @import("row_cuda_records.zig").cuda ++ @import("flashnext_tile_records.zig").cuda ++ @import("window_rows_records.zig").cuda ++ @import("modelopt_lane_records.zig").cuda ++ @import("nvfp4_fact_records.zig").cuda ++ @import("lane_half_records.zig").cuda ++ @import("pocket_records.zig").cuda ++ @import("minimax_cuda_records.zig").cuda;
+} ++ @import("registry_media.zig").cuda ++ @import("nemotron_prompt_records.zig").cuda ++ @import("glm_cuda_records.zig").cuda ++ @import("registry_shared_attention.zig").records(CudaRecord) ++ @import("gemma_cuda_records.zig").cuda ++ @import("spark_cuda_records.zig").cuda ++ @import("row_cuda_records.zig").cuda ++ @import("flashnext_tile_records.zig").cuda ++ @import("window_rows_records.zig").cuda ++ @import("modelopt_lane_records.zig").cuda ++ @import("nvfp4_fact_records.zig").cuda ++ @import("lane_half_records.zig").cuda ++ @import("pocket_records.zig").cuda ++ @import("minimax_cuda_records.zig").cuda ++ @import("exl3_dense_records.zig").cuda;
 
 /// The nvcc release `cuda`'s usage records come from: the compiler that builds the release fatbins.
 pub const cuda_release: limits.Nvcc = .{ .major = 13, .minor = 0 };
diff --git a/zig/src/cuda/exl3.zig b/zig/src/cuda/exl3.zig
index bfeee976a1..da2a811e7b 100644
--- a/zig/src/cuda/exl3.zig
+++ b/zig/src/cuda/exl3.zig
@@ -3,6 +3,7 @@ const std = @import("std");
 const candidate = @import("exl3_candidate.zig");
 const platform = @import("sys");
 const driver = @import("driver.zig");
+const abi = @import("abi.zig");
 const module = @import("module.zig");
 const launch_ = @import("launch.zig");
 const stream_ = @import("stream.zig");
@@ -174,6 +175,14 @@ pub fn planOf(w: Weight) struct { sk: u32, wk: u32 } {
     return .{ .sk = p.sk, .wk = p.wk };
 }
 
+/// Whether the current context's device takes programmatic dependent launches (compute capability 9.0 or newer).
+fn dependentLaunches(d: *const Driver) bool {
+    var dev: abi.Device = 0;
+    var major: c_int = 0;
+    if (d.api.cuCtxGetDevice(&dev) != abi.success) return false;
+    return d.api.cuDeviceGetAttribute(&major, .compute_capability_major, dev) == abi.success and major >= 9;
+}
+
 /// The module and its kernels.
 pub const Linear = struct {
     mod: Module,
@@ -207,6 +216,7 @@ pub const Linear = struct {
         }
         l.group = .{ try l.mod.function("tf_exl3_group_8_2_4"), try l.mod.function("tf_exl3_group_8_2_8") };
         l.rot_group = try l.mod.function("tf_exl3_rot_group");
+        l.pdl = dependentLaunches(d); // sm_90 and newer; the kernels' waits compile to nothing below
         if (platform.c.getenv("TF_EXL3_PDL")) |v| l.pdl = !std.mem.eql(u8, std.mem.span(v), "0");
         return l;
     }

From a797511fa8f671f87b0f322146baf334bd8e765e Mon Sep 17 00:00:00 2001
From: BHCC2025 <241330250+BHCC2025@users.noreply.github.com>
Date: Tue, 27 Oct 2026 11:04:01 -0500
Subject: [PATCH 4/8] exl3 cuda tests: the 27B's shapes against float64 under
 plan()'s launch and the fixed one (exl3-plan-f64)

---
 zig/tests/cuda/exl3_plan_bench.zig |  8 +--
 zig/tests/cuda/exl3_plan_f64.zig   | 97 ++++++++++++++++++++++++++++++
 zig/tests/cuda/exl3_tests.zig      |  7 +--
 zig/tests/cuda/main.zig            |  3 +
 4 files changed, 107 insertions(+), 8 deletions(-)
 create mode 100644 zig/tests/cuda/exl3_plan_f64.zig

diff --git a/zig/tests/cuda/exl3_plan_bench.zig b/zig/tests/cuda/exl3_plan_bench.zig
index d074a7930f..64afe1ad0b 100644
--- a/zig/tests/cuda/exl3_plan_bench.zig
+++ b/zig/tests/cuda/exl3_plan_bench.zig
@@ -8,10 +8,10 @@ const exl3 = cuda.exl3;
 const Gpu = check.Gpu;
 const Rig = quant_ref_tests.RigT;
 
-const Shape = struct { name: []const u8, k: usize, n: usize, halves: u32 };
+pub const Shape = struct { name: []const u8, k: usize, n: usize, halves: u32 };
 
 /// Qwen3.8-27B's projections in turboderp's 4.00bpw pack (6-bit head).
-const shapes = [_]Shape{
+pub const shapes = [_]Shape{
     .{ .name = "gate_up", .k = 5120, .n = 17408, .halves = 8 },
     .{ .name = "down", .k = 17408, .n = 5120, .halves = 8 },
     .{ .name = "qkv", .k = 5120, .n = 10240, .halves = 8 },
@@ -102,8 +102,8 @@ fn shape(r: *Rig, lin: *const exl3.Linear, sh: Shape) !void {
     }
 }
 
-/// The family's fixed plans (cuda_weights_exl3.zig) by shape name, for `midm`.
-const fixed = [_]struct { name: []const u8, sk: u32, wk: u32 }{
+/// The family's fixed plans (cuda_exl3_weights.zig) by shape name, for `midm` and `exl3-plan-f64`.
+pub const fixed = [_]struct { name: []const u8, sk: u32, wk: u32 }{
     .{ .name = "gate_up", .sk = 1, .wk = 4 }, .{ .name = "down", .sk = 1, .wk = 8 },  .{ .name = "qkv", .sk = 1, .wk = 4 },
     .{ .name = "z", .sk = 2, .wk = 4 },       .{ .name = "out_o", .sk = 2, .wk = 8 }, .{ .name = "q", .sk = 1, .wk = 8 },
     .{ .name = "k_v", .sk = 5, .wk = 8 },     .{ .name = "head", .sk = 1, .wk = 4 },
diff --git a/zig/tests/cuda/exl3_plan_f64.zig b/zig/tests/cuda/exl3_plan_f64.zig
new file mode 100644
index 0000000000..6e8766c1cb
--- /dev/null
+++ b/zig/tests/cuda/exl3_plan_f64.zig
@@ -0,0 +1,97 @@
+//! `tf-cuda-test exl3-plan-f64`: the 27B's EXL3 shapes against float64, under plan()'s launch and the fixed one.
+const std = @import("std");
+const cuda = @import("cuda");
+const check = @import("check.zig");
+const quant_ref_tests = @import("quant_ref_tests.zig");
+const exl3_tests = @import("exl3_tests.zig");
+const bench = @import("exl3_plan_bench.zig");
+const exl3fmt = @import("core").exl3_format;
+const exl3 = cuda.exl3;
+const Gpu = check.Gpu;
+const Rig = quant_ref_tests.RigT;
+
+/// One launch's error against float64: the largest share of the bound, the worst column and the root mean square.
+const Stats = struct { q: f64 = 0, worst: f64 = 0, col: usize = 0, rms: f64 = 0 };
+
+pub fn run(gpu: Gpu, only: ?[]const u8) !void {
+    var arena = std.heap.ArenaAllocator.init(gpu.gpa);
+    defer arena.deinit();
+    var s = try cuda.Stream.init(gpu.d, false);
+    defer s.deinit();
+    var prng = std.Random.DefaultPrng.init(13);
+    var r: Rig = .{ .gpu = gpu, .a = arena.allocator(), .s = s, .rng = prng.random() };
+    defer {
+        for (r.bufs.items) |*b| b.free();
+        r.bufs.deinit(gpu.gpa);
+    }
+    var lin = try exl3.Linear.load(gpu.d);
+    defer lin.unload();
+    var bad: usize = 0;
+    for (bench.shapes, bench.fixed) |sh, f| {
+        if (only) |o| if (!std.mem.eql(u8, o, sh.name)) continue;
+        bad += try shape(&r, &lin, sh, f.sk, f.wk);
+        for (r.bufs.items) |*b| b.free();
+        r.bufs.clearRetainingCapacity();
+    }
+    try check.expect(bad == 0, "exl3.plan.f64: {d} launches outside float64's bound", .{bad});
+    check.pass("exl3.plan.f64", .{});
+}
+
+/// Decode rows (16) and prompt rows (128) on identical inputs, each plan's fp32 out against one float64 reference.
+fn shape(r: *Rig, lin: *const exl3.Linear, sh: bench.Shape, sk: u32, wk: u32) !usize {
+    const width = exl3fmt.Width{ .halves = @intCast(sh.halves) };
+    const rows: usize = if (sh.n > 65536) 16 else 128; // the head scores decode rows and verify windows only
+    const trellis = try r.a.alloc(i16, (sh.k / 16) * (sh.n / 16) * width.tileWords());
+    for (trellis) |*v| v.* = r.rng.int(i16);
+    const suh = try r.a.alloc(u16, sh.k);
+    const svh = try r.a.alloc(u16, sh.n);
+    for (suh) |*v| v.* = @bitCast(@as(f16, @floatCast(r.rng.float(f32) * 2 - 1)));
+    for (svh) |*v| v.* = @bitCast(@as(f16, @floatCast(r.rng.float(f32) * 2 - 1)));
+    const x = try r.bfs(rows * sh.k, 1.0);
+    const ref = try exl3_tests.reference(r, width, .mul1, trellis, sh.k, sh.n, x, suh, svh, rows);
+    const st = exl3.strides(sh.k, sh.n, sh.halves, false);
+    const base: exl3.Weight = .{ .trellis = try r.dev(std.mem.sliceAsBytes(trellis)), .suh = try r.dev(std.mem.sliceAsBytes(suh)), .svh = try r.dev(std.mem.sliceAsBytes(svh)), .k = @intCast(sh.k), .n = @intCast(sh.n), .halves = sh.halves, .codebook = .mul1, .stride_k = st.k, .stride_nb = st.nb };
+    var tuned = base;
+    tuned.sk = sk;
+    tuned.wk = wk;
+    const dx = try r.dev(std.mem.sliceAsBytes(x));
+    const xh = try r.zeros(rows * sh.k * 2);
+    const z = try r.zeros(@max(exl3.Linear.zBytes(base, rows), exl3.Linear.zBytes(tuned, rows)) + 16);
+    const counters = try exl3_tests.zeroed(r, exl3.Linear.counterBytes(base));
+    const y = try r.zeros(rows * sh.n * 4);
+    var bad: usize = 0;
+    const ms: []const usize = if (rows == 16) &.{16} else &.{ 16, 128 };
+    for (ms) |m| {
+        var got: [2]Stats = undefined;
+        var outs: [2][]u8 = undefined;
+        for ([_]exl3.Weight{ base, tuned }, &got, &outs, [_][]const u8{ "plan()", "fixed" }) |w, *g, *o, name| {
+            try lin.matmul(r.s, w, dx, .bf16, m, y, .f32, xh, z, counters);
+            g.* = measure(try r.back(y, m * sh.n * 4), ref.want, ref.tol, m, sh.n);
+            if (g.q > 1) bad += 1;
+            try lin.matmul(r.s, w, dx, .bf16, m, y, .bf16, xh, z, counters);
+            o.* = try r.back(y, m * sh.n * 2);
+            const at = exl3.planOf(w);
+            std.debug.print("RESULT exl3.plan.f64 {s} {s} m={d} {s} sk={d} wk={d}: fp32 out worst {e:.3} (column {d}), rms {e:.3}, {d:.3} of the bound\n", .{ sh.name, if (m == 16) "decode" else "prompt", m, name, at.sk, at.wk, g.worst, g.col, g.rms, g.q });
+        }
+        var differ: usize = 0;
+        for (0..m * sh.n) |i| differ += @intFromBool(!std.mem.eql(u8, outs[0][2 * i ..][0..2], outs[1][2 * i ..][0..2]));
+        std.debug.print("RESULT exl3.plan.f64 {s} m={d}: fixed / plan() fp32 rms {d:.4}, worst {d:.4}; bf16 outputs that differ {d} of {d}\n", .{ sh.name, m, got[1].rms / got[0].rms, got[1].worst / got[0].worst, differ, m * sh.n });
+    }
+    return bad;
+}
+
+fn measure(raw: []const u8, want: []const f64, tol: []const f64, m: usize, n: usize) Stats {
+    var s: Stats = .{};
+    var sq: f64 = 0;
+    for (0..m * n) |i| {
+        const e = @abs(@as(f64, std.mem.bytesToValue(f32, raw[4 * i ..][0..4])) - want[i]);
+        s.q = @max(s.q, e / tol[i]);
+        sq += e * e;
+        if (e > s.worst) {
+            s.worst = e;
+            s.col = i % n;
+        }
+    }
+    s.rms = @sqrt(sq / @as(f64, @floatFromInt(m * n)));
+    return s;
+}
diff --git a/zig/tests/cuda/exl3_tests.zig b/zig/tests/cuda/exl3_tests.zig
index d16294c179..0389f0863c 100644
--- a/zig/tests/cuda/exl3_tests.zig
+++ b/zig/tests/cuda/exl3_tests.zig
@@ -134,7 +134,7 @@ fn f64Linear(r: *Rig, lin: *const exl3.Linear, c: Cfg) !void {
     for (suh) |*v| v.* = @bitCast(@as(f16, @floatCast(r.rng.float(f32) * 2 - 1)));
     for (svh) |*v| v.* = @bitCast(@as(f16, @floatCast(r.rng.float(f32) * 2 - 1)));
     const x = try r.bfs(128 * k, 1.0);
-    const want = try reference(r, width, c.codebook, trellis, k, n, x, suh, svh);
+    const want = try reference(r, width, c.codebook, trellis, k, n, x, suh, svh, 128);
     const st = exl3.strides(k, n, c.halves, false);
     const w: exl3.Weight = .{
         .trellis = try r.dev(std.mem.sliceAsBytes(trellis)),
@@ -159,9 +159,8 @@ fn f64Linear(r: *Rig, lin: *const exl3.Linear, c: Cfg) !void {
     }
 }
 
-/// The Python engine's f64 reference: Wq fp16 from the trellis, fp16 xh, the y sums, and each element's |t| sum.
-fn reference(r: *Rig, width: exl3fmt.Width, cb: exl3fmt.Codebook, trellis: []const i16, k: usize, n: usize, x: []const u16, suh: []const u16, svh: []const u16) !struct { want: []f64, tol: []f64 } {
-    const m = 128;
+/// The Python engine's f64 reference for m (at most 128) rows: Wq fp16 from the trellis, fp16 xh, y, each |t| sum.
+pub fn reference(r: *Rig, width: exl3fmt.Width, cb: exl3fmt.Codebook, trellis: []const i16, k: usize, n: usize, x: []const u16, suh: []const u16, svh: []const u16, m: usize) !struct { want: []f64, tol: []f64 } {
     const wq = try r.a.alloc(f16, k * n);
     exl3fmt.unpack(width, cb, trellis, k / 16, n / 16, wq);
     const xs = try r.a.alloc(f64, m * k);
diff --git a/zig/tests/cuda/main.zig b/zig/tests/cuda/main.zig
index 556e2a0bf2..3c37e4170b 100644
--- a/zig/tests/cuda/main.zig
+++ b/zig/tests/cuda/main.zig
@@ -42,6 +42,7 @@ const modelopt_tests = @import("modelopt_tests.zig");
 const exl3_tests = @import("exl3_tests.zig");
 const exl3_tp2_tests = @import("exl3_tp2_tests.zig");
 const exl3_plan_bench = @import("exl3_plan_bench.zig");
+const exl3_plan_f64 = @import("exl3_plan_f64.zig");
 const exl3_candidate_tests = @import("exl3_candidate_tests.zig");
 const experts3_tests = @import("experts3_tests.zig");
 const moe_tests = @import("moe_tests.zig");
@@ -228,6 +229,7 @@ const usage =
     \\  exl3-shapes [rows]        the 27B's EXL3 shapes once each with the fixed plans (1 and 16 rows), for ncu
     \\  exl3-midm                 the mid-M kernels, linear_wc and the projection groups against the linear, byte for byte
     \\  exl3-plans [shape]        the 27B's EXL3 shapes: every (K splits, warps) and layout timed at 1 and 16 rows
+    \\  exl3-plan-f64 [shape]     the 27B's EXL3 shapes against float64 under plan()'s launch and the fixed one, 16 and 128 rows
     \\  exl3-tp2 <rank> <host:port> [dir]  EXL3 tensor parallel: rank shards vs the Python engine's, all-gathered (dir none skips them)
     \\  exl3-tiles [DIR]          EXL3's tile candidate: codebooks and tiles bit-equal, FP64, rows through qlinear and Dense
     \\  train-pack MODEL EXAMPLES [COUNT]   training passes holding several sequences against one a pass: differences, time
@@ -399,6 +401,7 @@ fn run(gpu: check.Gpu, cmd: []const u8, rest: []const [:0]const u8) !void {
     if (std.mem.eql(u8, cmd, "exl3-shapes")) return exl3_plan_bench.profileShapes(gpu, if (rest.len > 0) rest[0] else null);
     if (std.mem.eql(u8, cmd, "exl3-midm")) return exl3_plan_bench.midm(gpu);
     if (std.mem.eql(u8, cmd, "exl3-plans")) return exl3_plan_bench.run(gpu, if (rest.len > 0) rest[0] else null);
+    if (std.mem.eql(u8, cmd, "exl3-plan-f64")) return exl3_plan_f64.run(gpu, if (rest.len > 0) rest[0] else null);
     if (std.mem.eql(u8, cmd, "exl3-tp2")) return exl3_tp2_tests.run(gpu, try arg(rest, 0), try arg(rest, 1), if (rest.len > 2) rest[2] else "none");
     if (std.mem.eql(u8, cmd, "sample")) return sample_tests.draws(gpu);
     if (std.mem.eql(u8, cmd, "topk")) return sample_tests.topk(gpu);

From 1c1ff3335a98ca0bc0d60d47c64e60b273f57469 Mon Sep 17 00:00:00 2001
From: BHCC2025 <241330250+BHCC2025@users.noreply.github.com>
Date: Tue, 27 Oct 2026 19:33:45 -0500
Subject: [PATCH 5/8] exl3 cuda: review fixes - plan()'s launches, group
 reserves after the MLP, group counters through grownCounters, the
 replacements in the kernel contract

- The 27B's fixed plans and exl3-plan-f64 are gone: every projection takes plan()'s launch (Weight.sk/wk stay).
- reserveGroups runs after the layer's MLP is set; a host test checks a dense layer's gate/up group decodes 1-16 rows
  without allocating.
- A group's counters come from main's grownCounters (#1475) with its members' summed bytes; scratch() never zeroes.
- contract_exl3.zig registers the mid-M kernels, linear_wc, the groups and rot_group in gates 1-4 as byte-equal
  replacements of linear_body (rot_in), each with a planted control; host tests for groupable, wcPick and planOf.
---
 docs/kernel-contract.md                       |   5 +
 zig/src/cuda/exl3.zig                         |  42 +-
 zig/src/cuda/exl3_dense_test.zig              |  86 ++++
 .../families/qwen3_5/cuda_exl3_weights.zig    |  25 +-
 zig/tests/cuda/contract_exl3.zig              | 445 ++++++++++++++++++
 zig/tests/cuda/contract_tests.zig             |   4 +
 zig/tests/cuda/exl3_plan_bench.zig            |  26 +-
 zig/tests/cuda/exl3_plan_f64.zig              |  97 ----
 zig/tests/cuda/exl3_tests.zig                 |   7 +-
 zig/tests/cuda/main.zig                       |   8 +-
 10 files changed, 582 insertions(+), 163 deletions(-)
 create mode 100644 zig/src/cuda/exl3_dense_test.zig
 create mode 100644 zig/tests/cuda/contract_exl3.zig
 delete mode 100644 zig/tests/cuda/exl3_plan_f64.zig

diff --git a/docs/kernel-contract.md b/docs/kernel-contract.md
index 92a3ca8ad4..ff170d9457 100644
--- a/docs/kernel-contract.md
+++ b/docs/kernel-contract.md
@@ -183,6 +183,11 @@ CUDA also checks Dory's activation rows (`act_l2.cuh`): the scaled sigmoid outpu
 each row then L2-normed, against float64's ranges at their bf16 points and expf's ulps (four controls, one point each).
 CUDA also checks `dense_rows`' scaled epilogue (Dory's logits, fp32(bf16(x . w)) times a per-column scale), bit-equal to its
 unscaled tile's outputs rounded and scaled on the host.
+CUDA also checks EXL3's replacements of `linear_body` (`exl3_linear.cu`), each byte-equal to it at every row count it
+takes: the mid-M kernels and `linear_wc` (4- and 6-bit, 1-128 rows; `Linear.min_rows` 1 runs them below 17 rows), the
+projection groups against each member's own launch, and `tf_exl3_rot_group` against `tf_exl3_rot_in` (1-16 rows). Gate 4
+proves an fp16 sum would overflow on a constant-weight stress input. Four controls: another K split, swapped scales, a
+swapped suh.
 HIP: the RMS norm rows, a wave a row (fp32, fp16 and bf16 out) and a block a row (fp16 and bf16 out).
 
 RADIO's fused bias, activation and residual (`tf_vit_bias_act`) replaces `tf_vit_bias` then
diff --git a/zig/src/cuda/exl3.zig b/zig/src/cuda/exl3.zig
index da2a811e7b..86368fb17a 100644
--- a/zig/src/cuda/exl3.zig
+++ b/zig/src/cuda/exl3.zig
@@ -85,10 +85,10 @@ fn wcShared(v: Wc) u32 {
 }
 
 /// linear.cu dispatch_wc: which linear_wc takes m rows of this weight, or null for the mid-M kernels.
-fn wcPick(w: Weight, m: usize, sk: u32, wk: u32) ?usize {
+pub fn wcPick(w: Weight, m: usize, sk: u32, wk: u32, min_rows: usize) ?usize {
     if (w.codebook != .mul1 or (w.halves != 8 and w.halves != 12)) return null;
     const per_warp = w.k / 16 / sk / wk;
-    if (per_warp % 2 != 0 or m <= 16 or m > 128) return null;
+    if (per_warp % 2 != 0 or m < min_rows or m > 128) return null;
     const nb = w.n / 128;
     const fold = sk > 1 and nb >= 32;
     if (!fold and w.halves == 8 and m > 32 and m <= 40) return null;
@@ -198,6 +198,8 @@ pub const Linear = struct {
     pdl: bool = true,
     /// 17-128 rows on the mid-M kernels where a width has them (false: the linear at every row count, same bits).
     midm: bool = true,
+    /// The least rows the mid-M kernels and linear_wc take (the kernel contract: 1, proving their bits at every count).
+    min_rows: usize = 17,
 
     pub fn load(d: *const Driver) !Linear {
         if (!kernels.available) return error.BuiltWithoutKernels;
@@ -282,7 +284,7 @@ pub const Linear = struct {
         args.add(@as(i32, @intCast(m)));
         args.add(@as(i32, @intCast(w.k)));
         // 17-48 rows: 16-row groups; 49-128: 32-row groups, each group's tiles decoded once (linear.cu's dispatch)
-        if (l.midm and l.use_wc) if (wcPick(w, m, p.sk, p.wk)) |wi| {
+        if (l.midm and l.use_wc) if (wcPick(w, m, p.sk, p.wk, l.min_rows)) |wi| {
             const v = wc_variants[wi];
             args.add(@as(i32, @intCast(p.wk))); // WcArgs: LinearArgs, then the plan's warps
             const rg = 16 * v.mt;
@@ -293,7 +295,7 @@ pub const Linear = struct {
             }, s, &args);
         };
         const pg: u32 = if (m > 48) 2 else 1;
-        if (l.midm and m > 16) if (mpgIndex(w.halves, w.codebook, p.wk, pg)) |mi| {
+        if (l.midm and m >= l.min_rows) if (mpgIndex(w.halves, w.codebook, p.wk, pg)) |mi| {
             const groups = (m + 16 * pg - 1) / (16 * pg);
             return launch_.launch(l.mpg[mi], .{
                 .grid = .{ .x = @intCast(w.n / 128 * groups), .y = p.sk, .z = 1 },
@@ -394,7 +396,7 @@ pub const Dense = struct {
 
     /// Grow every scratch buffer to what 128 rows of `w` need on `s`, so a call in a graph capture allocates nothing.
     pub fn reserve(de: *Dense, s: Stream, w: Weight) !void {
-        if (planOf(w).sk > 1) _ = try de.scratch(&de.z, Linear.zBytes(w, 128)); // a fixed plan's splits too
+        if (planOf(w).sk > 1) _ = try de.scratch(&de.z, Linear.zBytes(w, 128));
         _ = try de.grownCounters(s, Linear.counterBytes(w));
         _ = try de.scratch(&de.xh, 128 * w.k * 2);
     }
@@ -409,16 +411,22 @@ pub const Dense = struct {
 
     /// `reserve` for a group `decodeGroup` launches at once: its members' scratch side by side, at 16 rows.
     pub fn reserveGroup(de: *Dense, s: Stream, ws: []const Weight) !void {
-        if (!groupable(ws, 16)) return;
+        if (groupable(ws, 16)) _ = try de.groupScratch(s, ws, 16);
+    }
+
+    /// A group's scratch at `rows`: its members' split partials, counters and rotated inputs side by side.
+    pub fn groupScratch(de: *Dense, s: Stream, ws: []const Weight, rows: usize) !struct { z: u64, counters: u64, xh: u64 } {
         var zb: usize = 0;
         var cb: usize = 0;
         for (ws) |w| {
             zb += Linear.zBytes(w, 128);
             cb += Linear.counterBytes(w);
         }
-        if (zb > 0) _ = try de.scratch(&de.z, zb);
-        _ = try de.grownCounters(s, cb);
-        _ = try de.scratch(&de.xh, ws.len * 16 * ws[0].k * 2);
+        return .{
+            .z = if (zb > 0) try de.scratch(&de.z, zb) else 0,
+            .counters = try de.grownCounters(s, cb),
+            .xh = try de.scratch(&de.xh, ws.len * rows * ws[0].k * 2),
+        };
     }
 
     /// Decode rows: one call (the kernel takes 1..128), any count above in 128-row blocks.
@@ -465,16 +473,8 @@ pub const Dense = struct {
             for (ws, ys) |w, y| try de.decode(s, w, x, x_dtype, rows, y, y_dtype);
             return;
         }
-        var zb: usize = 0;
-        var cb: usize = 0;
-        for (ws) |w| {
-            zb += Linear.zBytes(w, 128);
-            cb += Linear.counterBytes(w);
-        }
-        const z: u64 = if (zb > 0) try de.scratch(&de.z, zb) else 0;
-        const counters = try de.grownCounters(s, cb); // the members' counters back to back
-        const xh = try de.scratch(&de.xh, ws.len * rows * ws[0].k * 2);
-        try de.lin.matmulGroup(s, ws, x, x_dtype, rows, ys, y_dtype, xh, z, counters);
+        const sc = try de.groupScratch(s, ws, rows);
+        try de.lin.matmulGroup(s, ws, x, x_dtype, rows, ys, y_dtype, sc.xh, sc.z, sc.counters);
     }
 
     fn blocks(de: *Dense, s: Stream, w: Weight, x: u64, x_dtype: DType, rows: usize, y: u64, y_dtype: DType) !void {
@@ -597,3 +597,7 @@ test "candidate selection refuses layouts it cannot represent" {
 }
 
 pub const block = @import("exl3_block.zig");
+
+test {
+    _ = @import("exl3_dense_test.zig");
+}
diff --git a/zig/src/cuda/exl3_dense_test.zig b/zig/src/cuda/exl3_dense_test.zig
new file mode 100644
index 0000000000..4e24837632
--- /dev/null
+++ b/zig/src/cuda/exl3_dense_test.zig
@@ -0,0 +1,86 @@
+//! Host tests of the EXL3 launch choices (planOf, wcPick, groupable) and of Dense's scratch: no device work.
+const std = @import("std");
+const exl3 = @import("exl3.zig");
+const mock = @import("exl3_mock.zig");
+const memory = @import("memory.zig");
+const Stream = @import("stream.zig").Stream;
+const Weight = exl3.Weight;
+const expectEqual = std.testing.expectEqual;
+
+fn weight(k: u32, n: u32, halves: u32) Weight {
+    const st = exl3.strides(k, n, halves, false);
+    return .{ .trellis = 1, .suh = 2, .svh = 3, .k = k, .n = n, .halves = halves, .codebook = .mul1, .stride_k = st.k, .stride_nb = st.nb };
+}
+
+test "planOf takes plan()'s launch unless the weight fixes one" {
+    var w = weight(4096, 1024, 8);
+    try expectEqual(exl3.plan(4096, 1024, 192, 8).sk, exl3.planOf(w).sk);
+    try expectEqual(@as(u32, 8), exl3.planOf(w).sk);
+    try expectEqual(@as(u32, 4), exl3.planOf(w).wk);
+    w.sk = 5;
+    w.wk = 8;
+    try expectEqual(@as(u32, 5), exl3.planOf(w).sk);
+    try expectEqual(@as(u32, 8), exl3.planOf(w).wk);
+}
+
+test "wcPick: 4- and 6-bit mul1 rows from min_rows to 128, the tile linear.cu's dispatch_wc names" {
+    const w = weight(5120, 17408, 8); // plan(): 1 split, 8 warps, 40 k tiles a warp
+    const p = exl3.planOf(w);
+    try expectEqual(@as(?usize, 1), exl3.wcPick(w, 17, p.sk, p.wk, 17)); // 32-row tiles, 4 k steps a stage
+    try expectEqual(@as(?usize, 3), exl3.wcPick(w, 128, p.sk, p.wk, 17)); // 64-row tiles
+    try expectEqual(@as(?usize, null), exl3.wcPick(w, 36, p.sk, p.wk, 17)); // 33-40 rows, 4-bit: mid-M
+    try expectEqual(@as(?usize, null), exl3.wcPick(w, 16, p.sk, p.wk, 17));
+    try expectEqual(@as(?usize, 1), exl3.wcPick(w, 16, p.sk, p.wk, 1)); // (the contract's runs)
+    try expectEqual(@as(?usize, null), exl3.wcPick(w, 129, p.sk, p.wk, 1));
+    var odd = weight(1920, 1024, 8);
+    try expectEqual(@as(?usize, null), exl3.wcPick(odd, 64, 5, 8, 17)); // 3 k tiles a warp: odd
+    odd.halves = 7;
+    try expectEqual(@as(?usize, null), exl3.wcPick(odd, 64, 1, 4, 17));
+    var mcg = w;
+    mcg.codebook = .mcg;
+    try expectEqual(@as(?usize, null), exl3.wcPick(mcg, 64, p.sk, p.wk, 17));
+}
+
+test "groupable: two or three 4-bit mul1 layers of one K and one warp count, 1 to 16 rows" {
+    const gate = weight(5120, 17408, 8);
+    try std.testing.expect(exl3.groupable(&.{ gate, gate }, 1));
+    try std.testing.expect(exl3.groupable(&.{ gate, gate }, 16));
+    try std.testing.expect(!exl3.groupable(&.{ gate, gate }, 17));
+    try std.testing.expect(!exl3.groupable(&.{ gate, gate }, 0));
+    try std.testing.expect(!exl3.groupable(&.{gate}, 8));
+    try std.testing.expect(!exl3.groupable(&.{ gate, gate, gate, gate }, 8));
+    try std.testing.expect(exl3.groupable(&.{ weight(5120, 12288, 8), gate, weight(5120, 10240, 8) }, 8)); // 8 warps
+    try std.testing.expect(exl3.groupable(&.{ weight(5120, 6144, 8), weight(5120, 1024, 8) }, 8)); // 4 warps, K split
+    try std.testing.expect(!exl3.groupable(&.{ weight(5120, 12288, 8), weight(5120, 1024, 8) }, 8)); // 8 warps and 4
+    try std.testing.expect(!exl3.groupable(&.{ gate, weight(5120, 17408, 12) }, 8)); // 6-bit
+    try std.testing.expect(!exl3.groupable(&.{ gate, weight(6144, 17408, 8) }, 8)); // another K
+}
+
+fn release(de: *exl3.Dense) void {
+    for ([_]*?memory.DeviceBuffer{ &de.xh, &de.z, &de.counters }) |slot| {
+        if (slot.* != null) slot.*.?.free();
+        slot.* = null;
+    }
+}
+
+test "after a dense layer's reserves, its gate/up group decodes 1-16 rows without allocating" {
+    var d = mock.init();
+    const s: Stream = .{ .d = &d, .handle = @ptrFromInt(0x5100) };
+    const ws = [_]Weight{ weight(5120, 17408, 8), weight(5120, 17408, 8) };
+    var de: exl3.Dense = .{ .d = &d, .lin = undefined };
+    defer release(&de);
+    for (ws) |w| try de.reserve(s, w); // what the loader does for each projection, then for the layer's groups
+    try de.reserveGroup(s, &ws);
+    const allocs = mock.allocations;
+    const fills = mock.memsets;
+    for (1..17) |rows| _ = try de.groupScratch(s, &ws, rows);
+    try expectEqual(allocs, mock.allocations);
+    try expectEqual(fills, mock.memsets);
+    // without the group's reserve the members' scratch is too small: the group would allocate (inside a capture)
+    var bare: exl3.Dense = .{ .d = &d, .lin = undefined };
+    defer release(&bare);
+    for (ws) |w| try bare.reserve(s, w);
+    const before = mock.allocations;
+    _ = try bare.groupScratch(s, &ws, 16);
+    try std.testing.expect(mock.allocations > before);
+}
diff --git a/zig/src/families/qwen3_5/cuda_exl3_weights.zig b/zig/src/families/qwen3_5/cuda_exl3_weights.zig
index 7e22a6cd10..97fc69c917 100644
--- a/zig/src/families/qwen3_5/cuda_exl3_weights.zig
+++ b/zig/src/families/qwen3_5/cuda_exl3_weights.zig
@@ -126,30 +126,11 @@ fn f32vec(s: *Sink, ck: *core.Checkpoint, comptime fmt: []const u8, args: anytyp
 const admit = @import("cuda_exl3_admit.zig");
 const group = admit.group;
 
-/// (K splits, warps) per Qwen3.8-27B projection (K, N, halves), chosen with tf-cuda-test exl3-plans.
-const plans = [_]struct { k: usize, n: usize, halves: u32, sk: u32, wk: u32 }{
-    .{ .k = 5120, .n = 17408, .halves = 8, .sk = 1, .wk = 4 }, // gate, up
-    .{ .k = 17408, .n = 5120, .halves = 8, .sk = 1, .wk = 8 }, // down
-    .{ .k = 5120, .n = 10240, .halves = 8, .sk = 1, .wk = 4 }, // in_proj_qkv
-    .{ .k = 5120, .n = 6144, .halves = 8, .sk = 2, .wk = 4 }, // in_proj_z
-    .{ .k = 6144, .n = 5120, .halves = 8, .sk = 2, .wk = 8 }, // out_proj, o_proj
-    .{ .k = 5120, .n = 12288, .halves = 8, .sk = 1, .wk = 8 }, // q_proj (8 warps: one group launch with k and v)
-    .{ .k = 5120, .n = 1024, .halves = 8, .sk = 5, .wk = 8 }, // k_proj, v_proj
-    .{ .k = 5120, .n = 248320, .halves = 12, .sk = 1, .wk = 4 }, // lm_head (6-bit)
-};
-
-/// A shape's fixed plan, so every row count keeps one reduction; other shapes take plan()'s.
-fn tuned(k: usize, n: usize, halves: u32) ?[2]u32 {
-    for (plans) |p| if (p.k == k and p.n == n and p.halves == halves) return .{ p.sk, p.wk };
-    return null;
-}
-
 /// An n-by-k EXL3 projection, its trellis kept as stored (Dense reads the stored layout).
 fn dense(s: *Sink, kit: ?*Kit, ck: *core.Checkpoint, prefix: []const u8, mod: []const u8, n: usize, k: usize) !Proj {
     var buf: [200]u8 = undefined;
     const gr = try admit.denseGroup(ck, try name(&buf, "{s}{s}", .{ prefix, mod }), n, k);
     const st = cuda.exl3.strides(k, n, gr.halves, false);
-    const fixed = tuned(k, n, gr.halves);
     const w: cuda.exl3.Weight = .{
         .trellis = try s.put(gr.trellis.bytes),
         .suh = try s.put(gr.suh.bytes),
@@ -160,8 +141,6 @@ fn dense(s: *Sink, kit: ?*Kit, ck: *core.Checkpoint, prefix: []const u8, mod: []
         .codebook = .mul1,
         .stride_k = st.k,
         .stride_nb = st.nb,
-        .sk = if (fixed) |f| f[0] else 0,
-        .wk = if (fixed) |f| f[1] else 0,
     };
     if (cuda.exl3.variantIndex(w.halves, w.codebook, cuda.exl3.planOf(w).wk) == null) return error.UnsupportedQwenQuantization;
     if (kit) |x| try x.dense.reserve(s.e.stream, w);
@@ -299,17 +278,17 @@ fn all(s: *Sink, ck: *core.Checkpoint) !void {
                 .k_norm = try norm1(s, ck, "{s}self_attn.k_norm.weight", .{p}, geometry.head_dim),
             } };
         }
-        if (kit) |k| try reserveGroups(s.e.stream, k, l.*);
         l.mlp = if (g.moe()) .{ .moe = .{ .router = try router(s, ck, p), .experts = try experts(s, ck, p) } } else .{ .dense = .{
             .gate = try dense(s, kit, ck, p, "mlp.gate_proj", g.intermediate, g.hidden),
             .up = try dense(s, kit, ck, p, "mlp.up_proj", g.intermediate, g.hidden),
             .down = try dense(s, kit, ck, p, "mlp.down_proj", g.hidden, g.intermediate),
         } };
+        if (kit) |k| try reserveGroups(k, s.e.stream, l.*);
     }
 }
 
 /// Grouped decode launches (q/k/v, qkv/z, gate/up) need their members' scratch together before any graph capture.
-fn reserveGroups(st: cuda.Stream, k: *Kit, l: engine.Layer) !void {
+pub fn reserveGroups(k: *Kit, st: cuda.Stream, l: engine.Layer) !void {
     switch (l.mixer) {
         .delta => |m| try k.dense.reserveGroup(st, &.{ m.qkv.x3, m.z.x3 }),
         .attention => |m| try k.dense.reserveGroup(st, &.{ m.q.x3, m.k.x3, m.v.x3 }),
diff --git a/zig/tests/cuda/contract_exl3.zig b/zig/tests/cuda/contract_exl3.zig
new file mode 100644
index 0000000000..e83ce36e36
--- /dev/null
+++ b/zig/tests/cuda/contract_exl3.zig
@@ -0,0 +1,445 @@
+//! EXL3's replacements of linear_body under the contract on CUDA: the mid-M kernels, linear_wc, the groups, rot_group.
+const std = @import("std");
+const cuda = @import("cuda");
+const core = @import("core");
+const check = @import("check.zig");
+const rig_ = @import("contract_rig.zig");
+
+const contract = core.contract;
+const c = contract.spec;
+const g = contract.gates;
+const exl3 = cuda.exl3;
+const fmt = core.exl3_format;
+
+/// What a registration launches; a control poses as its kind with one planted fault and must go red at gate 3.
+pub const Kind = enum { midm, wc, group, rot_group };
+
+const why = "the K sums of decoded fp16 weights: no input makes them exact in fp32, so gate 4 proves fp16's miss (an overflow) alone";
+const rule = "linear_body's bytes on the same inputs: a NaN and both infinities in x reach the outputs as linear_body's do";
+
+/// A K sum in any order (gate 4 simulates a sequential one), then the warps', splits' and output rotation's adds.
+fn depth(s: c.Shape) usize {
+    return s.k + 64;
+}
+
+fn rotDepth(_: c.Shape) usize {
+    return 7 + 8;
+}
+
+/// The engine's shapes (Qwen3.8-27B's, K 5120 / 6144): plan()'s splits from 1 to 8, 4 and 8 warps.
+pub fn spec(kind: Kind, halves: u32, control: bool) c.Spec {
+    const rows = [3]c.Tile{ .{ .size = 16 }, .{}, .{} };
+    const sums: c.Sums = .{ .f16 = 4, .bf16 = 0, .exact = false, .why = why };
+    return switch (kind) {
+        .midm, .wc => .{
+            .name = switch (kind) {
+                .midm => if (control) "control: exl3 mid-M kernels at another K split posing as linear_body" else if (halves == 8) "cuda exl3 mid-M kernels tf_exl3_mpg (4-bit, 1-128 rows)" else "cuda exl3 mid-M kernels tf_exl3_mpg (6-bit, 1-128 rows)",
+                else => if (control) "control: exl3 linear_wc at another K split posing as linear_body" else if (halves == 8) "cuda exl3 linear_wc tf_exl3_wc (4-bit, 1-128 rows)" else "cuda exl3 linear_wc tf_exl3_wc (6-bit, 1-128 rows)",
+            },
+            .accum = .f32,
+            .depth = depth,
+            .ops = 4,
+            .out = .bf16,
+            .replaces = "tf_exl3_linear (linear_body)",
+            .nan = .{ .stated = rule },
+            .tiles = rows,
+            .served = if (halves == 8) &[_]c.Shape{ .{ .rows = 128, .n = 1024, .k = 5120 }, .{ .rows = 100, .n = 17408, .k = 5120 }, .{ .rows = 49, .n = 6144, .k = 5120 }, .{ .rows = 33, .n = 5120, .k = 6144 }, .{ .rows = 24, .n = 12288, .k = 5120 }, .{ .rows = 64, .n = 5120, .k = 6144 } } else &[_]c.Shape{ .{ .rows = 128, .n = 2048, .k = 5120 }, .{ .rows = 97, .n = 4096, .k = 5120 }, .{ .rows = 40, .n = 2048, .k = 5120 } },
+            .sums = sums,
+        },
+        .group => .{
+            .name = if (control) "control: exl3 group with a member's scales swapped posing as its members' linear_body" else "cuda exl3 projection groups tf_exl3_group_8_2_4 / _8 (1-16 rows)",
+            .accum = .f32,
+            .depth = depth,
+            .ops = 4,
+            .out = .bf16,
+            .replaces = "each member's tf_exl3_linear (linear_body)",
+            .nan = .{ .stated = rule },
+            .tiles = rows,
+            // n names the members: 17408 gate + up, 12288 q + gate + qkv (8 warps), 6144 z + k_v (4 warps, K split)
+            .served = &[_]c.Shape{ .{ .rows = 16, .n = 6144, .k = 5120 }, .{ .rows = 16, .n = 17408, .k = 5120 }, .{ .rows = 7, .n = 12288, .k = 5120 } },
+            .sums = sums,
+        },
+        .rot_group => .{
+            .name = if (control) "control: exl3 rot_group with a member's suh swapped posing as rot_in" else "cuda exl3 tf_exl3_rot_group (a group's rotated inputs, 1-16 rows)",
+            .accum = .f32,
+            .depth = rotDepth,
+            .ops = 2,
+            .out = .f16,
+            .replaces = "each member's tf_exl3_rot_in",
+            .tiles = rows,
+            .served = &[_]c.Shape{ .{ .rows = 16, .n = 6144, .k = 5120 }, .{ .rows = 16, .n = 17408, .k = 5120 }, .{ .rows = 7, .n = 12288, .k = 5120 } },
+            .sums = .{ .f16 = 4, .bf16 = 0, .exact = false, .why = "128-term rotations of bf16 inputs: fp32 is not exact on them, so gate 4 proves fp16's miss alone" },
+        },
+    };
+}
+
+/// A weight's host copy (trellis words, scales) and its device tensors.
+const Mat = struct {
+    k: usize,
+    n: usize,
+    trellis: []i16 = &.{},
+    suh: []u16 = &.{},
+    svh: []u16 = &.{},
+    w: exl3.Weight = undefined,
+};
+
+/// The members a group shape names (n): their output widths.
+fn members(n: usize) []const usize {
+    return switch (n) {
+        17408 => &.{ 17408, 17408 },
+        12288 => &.{ 12288, 17408, 10240 },
+        else => &.{ 6144, 1024 },
+    };
+}
+
+/// Another valid (K splits, warps) of the same shape: the controls' planted fault.
+fn other(w: exl3.Weight) exl3.Weight {
+    const p = exl3.planOf(w);
+    var x = w;
+    x.wk = if (p.wk == 4) 8 else 4;
+    x.sk = if (p.wk == 4) @max(1, p.sk / 2) else p.sk * 2;
+    return x;
+}
+
+pub const Case = struct {
+    rig: rig_.Rig,
+    lin: *exl3.Linear,
+    kind: Kind,
+    halves: u32 = 8,
+    control: bool = false,
+    shape: c.Shape = undefined,
+    mats: [3]Mat = .{ .{ .k = 0, .n = 0 }, .{ .k = 0, .n = 0 }, .{ .k = 0, .n = 0 } },
+    nm: usize = 1,
+    x: []u16 = &.{},
+    xd: u64 = 0,
+    out: u64 = 0,
+    base: u64 = 0,
+    xh: u64 = 0,
+    xh_base: u64 = 0,
+    z: u64 = 0,
+    counters: u64 = 0,
+    wbuf: [3]exl3.Weight = undefined,
+    msg: [160]u8 = undefined,
+
+    pub fn deinit(self: *Case) void {
+        self.free();
+        self.rig.deinit();
+    }
+
+    fn free(self: *Case) void {
+        const a = self.rig.gpu.gpa;
+        for (&self.mats) |*m| {
+            a.free(m.trellis);
+            a.free(m.suh);
+            a.free(m.svh);
+            m.* = .{ .k = 0, .n = 0 };
+        }
+        a.free(self.x);
+        self.x = &.{};
+    }
+
+    pub fn kernel(self: *Case) g.Kernel {
+        return .{ .spec = spec(self.kind, self.halves, self.control), .ctx = self, .vt = &vt, .control = if (self.control) .shapes else null };
+    }
+
+    const vt = g.hooks(Case);
+
+    fn random16(rng: std.Random, out: []u16) void {
+        for (out) |*v| v.* = @bitCast(@as(f16, @floatCast(rng.float(f32) * 2 - 1)));
+    }
+
+    /// Random trellis words and scales, x bf16 in [-1, 1); non-finite: x[1][3] NaN, x[2][5] +Inf, x[2][9] -Inf.
+    pub fn prepare(self: *Case, shape: c.Shape, fill: g.Fill, seed: u64) !bool {
+        const grouped = self.kind == .group or self.kind == .rot_group;
+        if (shape.rows == 0 or shape.rows > (if (grouped) @as(usize, 16) else 128) or shape.k % 128 != 0 or shape.n % 128 != 0) return false;
+        self.free();
+        self.rig.reset();
+        self.shape = shape;
+        const a = self.rig.gpu.gpa;
+        var prng = std.Random.DefaultPrng.init(seed);
+        const rng = prng.random();
+        const widths: []const usize = if (grouped) members(shape.n) else &.{shape.n};
+        self.nm = widths.len;
+        const width = fmt.Width{ .halves = @intCast(self.halves) };
+        var zb: usize = 0;
+        var cb: usize = 0;
+        for (widths, self.mats[0..widths.len]) |n, *m| {
+            m.k = shape.k;
+            m.n = n;
+            m.trellis = try a.alloc(i16, (shape.k / 16) * (n / 16) * width.tileWords());
+            m.suh = try a.alloc(u16, shape.k);
+            m.svh = try a.alloc(u16, n);
+            if (fill == .stress and self.kind != .rot_group) { // one weight everywhere: a 2-bit period, scales 1
+                @memset(m.trellis, @bitCast(@as(u16, 0xaaaa)));
+                @memset(m.suh, 0x3c00);
+                @memset(m.svh, 0x3c00);
+            } else {
+                for (m.trellis) |*v| v.* = rng.int(i16);
+                random16(rng, m.suh);
+                random16(rng, m.svh);
+            }
+            const st = exl3.strides(shape.k, n, self.halves, false);
+            m.w = .{ .trellis = try self.rig.input(std.mem.sliceAsBytes(m.trellis)), .suh = try self.rig.input(std.mem.sliceAsBytes(m.suh)), .svh = try self.rig.input(std.mem.sliceAsBytes(m.svh)), .k = @intCast(shape.k), .n = @intCast(n), .halves = self.halves, .codebook = .mul1, .stride_k = st.k, .stride_nb = st.nb };
+            zb += @max(exl3.Linear.zBytes(m.w, 128), exl3.Linear.zBytes(other(m.w), 128));
+            cb += exl3.Linear.counterBytes(m.w);
+        }
+        if (grouped and !exl3.groupable(self.weights(), shape.rows)) return false;
+        self.x = try a.alloc(u16, shape.rows * shape.k);
+        for (self.x) |*v| v.* = c.toBf16(rng.float(f32) * 2 - 1);
+        switch (fill) {
+            .random => {},
+            .nonfinite => if (shape.rows > 2) {
+                self.x[shape.k + 3] = 0x7fc0;
+                self.x[2 * shape.k + 5] = 0x7f80;
+                self.x[2 * shape.k + 9] = 0xff80;
+            },
+            .stress => if (self.kind == .rot_group) {
+                for (self.x) |*v| v.* = c.toBf16(c.fromBf16(v.*) * 64);
+            } else { // one input a block, rotated to 4096: K equal terms, past fp16's range
+                @memset(self.x, 0);
+                for (0..shape.rows) |r| for (0..shape.k / 128) |b| {
+                    self.x[r * shape.k + b * 128] = c.toBf16(46336);
+                };
+            },
+        }
+        self.xd = try self.rig.input(std.mem.sliceAsBytes(self.x));
+        self.out = try self.rig.output(self.outElems() * 2);
+        self.base = try self.rig.scratch(self.ysElems() * 2); // (linear_body's y; rot_group's group's y too)
+        self.xh = try self.rig.scratch(self.nm * shape.rows * shape.k * 2);
+        self.xh_base = try self.rig.scratch(self.nm * shape.rows * shape.k * 2);
+        self.z = try self.rig.scratch(@max(zb, 16));
+        self.counters = try self.rig.scratch(@max(cb, 16));
+        return true;
+    }
+
+    fn weights(self: *Case) []const exl3.Weight {
+        for (self.mats[0..self.nm], 0..) |m, i| self.wbuf[i] = m.w;
+        return self.wbuf[0..self.nm];
+    }
+
+    /// Output elements: the members' rows side by side (rot_group: their rotated inputs, fp16).
+    fn outElems(self: *const Case) usize {
+        return if (self.kind == .rot_group) self.nm * self.shape.rows * self.shape.k else self.ysElems();
+    }
+    fn ysElems(self: *const Case) usize {
+        var n: usize = 0;
+        for (self.mats[0..self.nm]) |m| n += m.n;
+        return self.shape.rows * n;
+    }
+
+    fn ys(self: *const Case, at: u64) [3]u64 {
+        var y: [3]u64 = .{ 0, 0, 0 };
+        var off: u64 = 0;
+        for (self.mats[0..self.nm], 0..) |m, i| {
+            y[i] = at + off;
+            off += self.shape.rows * m.n * 2;
+        }
+        return y;
+    }
+
+    pub fn run(self: *Case) !void {
+        const m = self.shape.rows;
+        switch (self.kind) {
+            .midm, .wc => {
+                self.lin.midm = true;
+                self.lin.use_wc = self.kind == .wc;
+                self.lin.min_rows = 1;
+                defer self.restore();
+                const w = if (self.control) other(self.mats[0].w) else self.mats[0].w;
+                try self.lin.matmul(self.rig.s, w, self.xd, .bf16, m, self.out, .bf16, self.xh, self.z, self.counters);
+            },
+            .group, .rot_group => {
+                var ws = self.weights();
+                var bad: [3]exl3.Weight = undefined;
+                if (self.control) {
+                    @memcpy(bad[0..self.nm], ws);
+                    if (self.kind == .group) bad[1].svh = bad[0].svh else bad[1].suh = bad[0].suh;
+                    ws = bad[0..self.nm];
+                }
+                const y = self.ys(if (self.kind == .group) self.out else self.base);
+                try self.lin.matmulGroup(self.rig.s, ws, self.xd, .bf16, m, y[0..self.nm], .bf16, if (self.kind == .rot_group) self.out else self.xh, self.z, self.counters);
+            },
+        }
+        try self.rig.s.synchronize();
+    }
+
+    fn restore(self: *Case) void {
+        self.lin.midm = true;
+        self.lin.use_wc = true;
+        self.lin.min_rows = 17;
+    }
+
+    /// linear_body on the same inputs, member by member (rot_group: each member's rot_in, the xh it leaves).
+    pub fn replaced(self: *Case) ![]const u8 {
+        self.lin.midm = false;
+        defer self.restore();
+        const m = self.shape.rows;
+        const y = self.ys(self.base);
+        for (self.mats[0..self.nm], 0..) |mat, i| {
+            const xh = self.xh_base + i * m * self.shape.k * 2;
+            try self.lin.matmul(self.rig.s, mat.w, self.xd, .bf16, m, y[i], .bf16, xh, self.z, self.counters);
+        }
+        try self.rig.s.synchronize();
+        if (self.kind == .rot_group) return self.rig.bytes(self.xh_base, self.outElems() * 2);
+        return self.rig.bytes(self.base, self.outElems() * 2);
+    }
+
+    /// Gate 1: linear_body's bytes, with some output non-finite to prove the planted values reach it.
+    pub fn check(self: *Case, _: g.Gate) !?[]const u8 {
+        const a = self.rig.gpu.gpa;
+        const got = try a.dupe(u8, try self.rig.outputBytes());
+        defer a.free(got);
+        const want = try self.replaced();
+        const s = spec(self.kind, self.halves, self.control);
+        if (std.mem.indexOfDiff(u8, got, want)) |at| return try std.fmt.bufPrint(&self.msg, "byte {d} of {d} differs from {s}'s", .{ at, got.len, s.replaces.? });
+        for (0..self.count()) |i| if (c.classOf(c.read(s.out, got, i)) != .finite) return null;
+        return try std.fmt.bufPrint(&self.msg, "no output is non-finite: the planted values prove nothing", .{});
+    }
+
+    pub fn count(self: *Case) usize {
+        return self.outElems();
+    }
+
+    /// Member i's x * suh rotated per 128 (rot_in's sums), each output's 128 terms summed in `acc` (null: float64).
+    fn rotated(self: *Case, i: usize, xh: []f64, mag: ?[]f64, acc: ?c.Accum, order: g.Order) !void {
+        const a = self.rig.gpu.gpa;
+        const k = self.shape.k;
+        const h = try a.alloc(f64, 128 * 128); // the rotation's matrix: row i the image of basis vector i
+        defer a.free(h);
+        @memset(h, 0);
+        for (0..128) |q| h[q * 128 + q] = 1;
+        fmt.rotate(h, 128, 128, 1);
+        var v: [128]f64 = undefined;
+        var t: [128]f64 = undefined;
+        for (0..self.shape.rows) |r| for (0..k / 128) |b| {
+            for (&v, 0..) |*e, q| e.* = c.fromBf16(self.x[r * k + b * 128 + q]) * @as(f64, @floatCast(@as(f16, @bitCast(self.mats[i].suh[b * 128 + q]))));
+            for (0..128) |j| {
+                var abs: f64 = 0;
+                for (&t, 0..) |*e, q| {
+                    e.* = h[q * 128 + j] * v[q];
+                    abs += @abs(e.*);
+                }
+                var sum: f64 = 0;
+                if (acc) |ac| {
+                    sum = contract.rows.sumIn(ac, order, &t);
+                } else {
+                    for (t) |e| sum += e;
+                }
+                xh[r * k + b * 128 + j] = sum;
+                if (mag) |m| m[r * k + b * 128 + j] = abs;
+            }
+        };
+    }
+
+    /// The rotated inputs the kernels take (linear_body's own rot_in's, member after member), fp16 values as float64.
+    fn kernelXh(self: *Case, i: usize, xs: []f64) !void {
+        _ = try self.replaced();
+        const n = self.shape.rows * self.shape.k;
+        const raw = try self.rig.bytes(self.xh_base + i * n * 2, n * 2);
+        for (xs, 0..) |*v, q| v.* = @floatCast(@as(f16, @bitCast(std.mem.bytesToValue(u16, raw[2 * q ..][0..2]))));
+    }
+
+    /// Float64 outputs and their terms' magnitude; with `acc`, the K sums simulated in that accumulator and order.
+    fn outputs(self: *Case, want: []f64, mag: ?[]f64, acc: ?c.Accum, order: g.Order) !void {
+        const a = self.rig.gpu.gpa;
+        const k = self.shape.k;
+        const rows = self.shape.rows;
+        const xs = try a.alloc(f64, rows * k);
+        defer a.free(xs);
+        var at: usize = 0;
+        for (0..self.nm) |i| {
+            if (self.kind == .rot_group) {
+                try self.rotated(i, want[at..][0 .. rows * k], if (mag) |mg| mg[at..][0 .. rows * k] else null, acc, order);
+                at += rows * k;
+                continue;
+            }
+            try self.kernelXh(i, xs); // (the K sums are the linear's: its inputs are rot_in's fp16 rows, as they ran)
+            const n = self.mats[i].n;
+            const wq = try a.alloc(f16, k * n);
+            defer a.free(wq);
+            fmt.unpack(.{ .halves = @intCast(self.halves) }, .mul1, self.mats[i].trellis, k / 16, n / 16, wq);
+            const ypre = try a.alloc(f64, rows * n);
+            defer a.free(ypre);
+            const s1 = try a.alloc(f64, rows * n);
+            defer a.free(s1);
+            const t = try a.alloc(f64, k);
+            defer a.free(t);
+            for (0..rows) |r| for (0..n) |j| {
+                var abs: f64 = 0;
+                for (0..k) |q| {
+                    t[q] = xs[r * k + q] * @as(f64, @floatCast(wq[q * n + j]));
+                    abs += @abs(t[q]);
+                }
+                var sum: f64 = 0;
+                if (acc) |ac| {
+                    sum = contract.rows.sumIn(ac, order, t);
+                } else {
+                    for (t) |v| sum += v;
+                }
+                ypre[r * n + j] = sum;
+                s1[r * n + j] = abs;
+            };
+            fmt.rotate(ypre, rows, n, 1);
+            for (0..rows) |r| for (0..n) |j| {
+                const sc: f64 = @floatCast(@as(f16, @bitCast(self.mats[i].svh[j])));
+                want[at + r * n + j] = ypre[r * n + j] * sc;
+                if (mag) |mg| {
+                    var s: f64 = 0;
+                    const b0 = j / 128 * 128;
+                    for (b0..b0 + 128) |q| s += s1[r * n + q];
+                    mg[at + r * n + j] = @abs(sc) * s / @sqrt(128.0);
+                }
+            };
+            at += rows * n;
+        }
+    }
+
+    pub fn reference(self: *Case, want: []f64, mag: []f64) !void {
+        return self.outputs(want, mag, null, .sequential);
+    }
+
+    pub fn simulate(self: *Case, acc: c.Accum, order: g.Order, out: []f64) !void {
+        return self.outputs(out, null, acc, order);
+    }
+};
+
+/// Every registration and control, on one Linear the cases share; `add` appends them to the runner's list.
+pub const Set = struct {
+    lin: ?exl3.Linear = null,
+    cases: [10]Case = undefined,
+    n: usize = 0,
+
+    pub fn add(self: *Set, gpu: check.Gpu, s: cuda.Stream, list: *std.ArrayList(g.Kernel)) !void {
+        self.lin = try exl3.Linear.load(gpu.d);
+        const all = [_]struct { k: Kind, h: u32 = 8, ctl: bool = false }{
+            .{ .k = .midm },               .{ .k = .midm, .h = 12 },          .{ .k = .wc },                .{ .k = .wc, .h = 12 },
+            .{ .k = .group },              .{ .k = .rot_group },              .{ .k = .midm, .ctl = true }, .{ .k = .wc, .ctl = true },
+            .{ .k = .group, .ctl = true }, .{ .k = .rot_group, .ctl = true },
+        };
+        for (all) |x| {
+            self.cases[self.n] = .{ .rig = .{ .gpu = gpu, .s = s }, .lin = &self.lin.?, .kind = x.k, .halves = x.h, .control = x.ctl };
+            self.n += 1;
+        }
+        for (self.cases[0..self.n]) |*cs| try list.append(gpu.gpa, cs.kernel());
+    }
+
+    pub fn deinit(self: *Set) void {
+        for (self.cases[0..self.n]) |*cs| cs.deinit();
+        if (self.lin) |*l| l.unload();
+    }
+};
+
+/// `tf-cuda-test exl3-contract`: these registrations' gates 1-4 and controls alone (kernel-contract runs them all).
+pub fn run(gpu: check.Gpu) !void {
+    var s = try cuda.Stream.init(gpu.d, false);
+    defer s.deinit();
+    var list: std.ArrayList(g.Kernel) = .empty;
+    defer list.deinit(gpu.gpa);
+    var set: Set = .{};
+    defer set.deinit();
+    try set.add(gpu, s, &list);
+    var t: g.Tally = .{};
+    try g.runAll(gpu.gpa, list.items, 0x5eed, &t);
+    std.debug.print("SUMMARY exl3 {d} kernels: {d} pass gates 1-4, {d} fail; {d} controls: {d} red as planted, {d} missed\n", .{ t.kernels, t.passed, t.failed, t.controls, t.caught, t.missed });
+    if (t.problems() > 0) return error.TestFailed;
+}
diff --git a/zig/tests/cuda/contract_tests.zig b/zig/tests/cuda/contract_tests.zig
index 812cd3dde4..f641dfbf15 100644
--- a/zig/tests/cuda/contract_tests.zig
+++ b/zig/tests/cuda/contract_tests.zig
@@ -9,6 +9,7 @@ const rows_k = @import("contract_rows.zig");
 const tail_k = @import("contract_norms_tail.zig");
 const tiles_k = @import("contract_rms_tiles.zig");
 const dory_k = @import("contract_dory.zig");
+const exl3_k = @import("contract_exl3.zig");
 const router_k = @import("contract_router.zig");
 const mm_k = @import("contract_mm.zig");
 const prefill_many_k = @import("contract_qmm_prefill_many.zig");
@@ -159,6 +160,9 @@ pub fn run(gpu: check.Gpu) !void {
     var dory: dory_k.Dory = .{};
     defer dory.deinit();
     try dory.add(gpu, s, &norms, mods[5], &kernels);
+    var exl3_set: exl3_k.Set = .{};
+    defer exl3_set.deinit();
+    try exl3_set.add(gpu, s, &kernels);
     var logits = router_k.Logits{ .rig = .{ .gpu = gpu, .s = s }, .f = try mods[0].function("tf_nemo_router") };
     defer logits.deinit();
     var logits_tiles = router_k.Logits{ .rig = .{ .gpu = gpu, .s = s }, .f = logits.f, .tiles = try mods[0].function("tf_nemo_router_tiles") };
diff --git a/zig/tests/cuda/exl3_plan_bench.zig b/zig/tests/cuda/exl3_plan_bench.zig
index 64afe1ad0b..3aca6879c8 100644
--- a/zig/tests/cuda/exl3_plan_bench.zig
+++ b/zig/tests/cuda/exl3_plan_bench.zig
@@ -102,13 +102,6 @@ fn shape(r: *Rig, lin: *const exl3.Linear, sh: Shape) !void {
     }
 }
 
-/// The family's fixed plans (cuda_exl3_weights.zig) by shape name, for `midm` and `exl3-plan-f64`.
-pub const fixed = [_]struct { name: []const u8, sk: u32, wk: u32 }{
-    .{ .name = "gate_up", .sk = 1, .wk = 4 }, .{ .name = "down", .sk = 1, .wk = 8 },  .{ .name = "qkv", .sk = 1, .wk = 4 },
-    .{ .name = "z", .sk = 2, .wk = 4 },       .{ .name = "out_o", .sk = 2, .wk = 8 }, .{ .name = "q", .sk = 1, .wk = 8 },
-    .{ .name = "k_v", .sk = 5, .wk = 8 },     .{ .name = "head", .sk = 1, .wk = 4 },
-};
-
 /// `tf-cuda-test exl3-midm`: the mid-M kernels against the linear at 17-128 rows, byte for byte, and both timed at 128.
 pub fn midm(gpu: Gpu) !void {
     var arena = std.heap.ArenaAllocator.init(gpu.gpa);
@@ -124,7 +117,7 @@ pub fn midm(gpu: Gpu) !void {
     var lin = try exl3.Linear.load(gpu.d);
     defer lin.unload();
     var bad: usize = 0;
-    for (shapes, fixed) |sh, f| {
+    for (shapes) |sh| {
         const words = (sh.k / 16) * (sh.n / 16) * sh.halves * 4;
         const host = try r.a.alloc(u32, words);
         for (host) |*v| v.* = r.rng.int(u32);
@@ -133,7 +126,7 @@ pub fn midm(gpu: Gpu) !void {
         for (suh) |*v| v.* = @bitCast(@as(f16, @floatCast(r.rng.float(f32) * 2 - 1)));
         for (svh) |*v| v.* = @bitCast(@as(f16, @floatCast(r.rng.float(f32) * 2 - 1)));
         const st = exl3.strides(sh.k, sh.n, sh.halves, false);
-        const w: exl3.Weight = .{ .trellis = try r.dev(std.mem.sliceAsBytes(host)), .suh = try r.dev(std.mem.sliceAsBytes(suh)), .svh = try r.dev(std.mem.sliceAsBytes(svh)), .k = @intCast(sh.k), .n = @intCast(sh.n), .halves = sh.halves, .codebook = .mul1, .stride_k = st.k, .stride_nb = st.nb, .sk = f.sk, .wk = f.wk };
+        const w: exl3.Weight = .{ .trellis = try r.dev(std.mem.sliceAsBytes(host)), .suh = try r.dev(std.mem.sliceAsBytes(suh)), .svh = try r.dev(std.mem.sliceAsBytes(svh)), .k = @intCast(sh.k), .n = @intCast(sh.n), .halves = sh.halves, .codebook = .mul1, .stride_k = st.k, .stride_nb = st.nb };
         const x = try r.dev(std.mem.sliceAsBytes(try r.bfs(128 * sh.k, 1.0)));
         const xh = try r.zeros(128 * sh.k * 2);
         const z = try r.zeros(exl3.Linear.zBytes(w, 128));
@@ -186,7 +179,7 @@ pub fn midm(gpu: Gpu) !void {
     check.pass("exl3.midm", .{});
 }
 
-/// `tf-cuda-test exl3-shapes [rows]`: each 27B shape once at its fixed plan, 1 and 16 rows, for a profiler to attach.
+/// `tf-cuda-test exl3-shapes [rows]`: each 27B shape once at plan()'s launch, 1 and 16 rows, for a profiler to attach.
 pub fn profileShapes(gpu: Gpu, rows_arg: ?[]const u8) !void {
     var arena = std.heap.ArenaAllocator.init(gpu.gpa);
     defer arena.deinit();
@@ -201,7 +194,7 @@ pub fn profileShapes(gpu: Gpu, rows_arg: ?[]const u8) !void {
     var lin = try exl3.Linear.load(gpu.d);
     defer lin.unload();
     const ms: []const usize = if (rows_arg) |a| &.{try std.fmt.parseInt(usize, a, 10)} else &.{ 1, 16 };
-    for (shapes, fixed) |sh, f| {
+    for (shapes) |sh| {
         const words = (sh.k / 16) * (sh.n / 16) * sh.halves * 4;
         const host = try r.a.alloc(u32, words);
         for (host) |*v| v.* = r.rng.int(u32);
@@ -210,14 +203,14 @@ pub fn profileShapes(gpu: Gpu, rows_arg: ?[]const u8) !void {
         @memset(suh, 0x3C00);
         @memset(svh, 0x3C00);
         const st = exl3.strides(sh.k, sh.n, sh.halves, false);
-        const w: exl3.Weight = .{ .trellis = try r.dev(std.mem.sliceAsBytes(host)), .suh = try r.dev(std.mem.sliceAsBytes(suh)), .svh = try r.dev(std.mem.sliceAsBytes(svh)), .k = @intCast(sh.k), .n = @intCast(sh.n), .halves = sh.halves, .codebook = .mul1, .stride_k = st.k, .stride_nb = st.nb, .sk = f.sk, .wk = f.wk };
+        const w: exl3.Weight = .{ .trellis = try r.dev(std.mem.sliceAsBytes(host)), .suh = try r.dev(std.mem.sliceAsBytes(suh)), .svh = try r.dev(std.mem.sliceAsBytes(svh)), .k = @intCast(sh.k), .n = @intCast(sh.n), .halves = sh.halves, .codebook = .mul1, .stride_k = st.k, .stride_nb = st.nb };
         const x = try r.dev(std.mem.sliceAsBytes(try r.bfs(128 * sh.k, 1.0)));
         const xh = try r.zeros(128 * sh.k * 2);
         const z = try r.zeros(exl3.Linear.zBytes(w, 128) + 16);
         const counters = try exl3_tests.zeroed(&r, exl3.Linear.counterBytes(w));
         const y = try r.zeros(128 * sh.n * 2);
         for (ms) |m| {
-            std.debug.print("SHAPE {s} m={d} sk={d} wk={d}\n", .{ sh.name, m, f.sk, f.wk });
+            std.debug.print("SHAPE {s} m={d} sk={d} wk={d}\n", .{ sh.name, m, exl3.planOf(w).sk, exl3.planOf(w).wk });
             for (0..2) |_| try lin.matmul(r.s, w, x, .bf16, m, y, .bf16, xh, z, counters);
             try r.s.synchronize();
         }
@@ -226,9 +219,9 @@ pub fn profileShapes(gpu: Gpu, rows_arg: ?[]const u8) !void {
     }
 }
 
-/// The 27B's projection groups (q k v, gate up, qkv z) in one launch against each member's own, byte for byte.
+/// Groups plan() admits (gate up; q, gate, qkv: 8 warps; z, k_v: 4, split K) in one launch against each member's own.
 fn groups(r: *Rig, lin: *const exl3.Linear) !usize {
-    const sets = [_][]const usize{ &.{ 5, 6, 6 }, &.{ 0, 0 }, &.{ 2, 3 } }; // indices into shapes / fixed
+    const sets = [_][]const usize{ &.{ 0, 0 }, &.{ 5, 0, 2 }, &.{ 3, 6 } }; // indices into shapes
     var bad: usize = 0;
     for (sets) |set| {
         var ws: [3]exl3.Weight = undefined;
@@ -236,7 +229,6 @@ fn groups(r: *Rig, lin: *const exl3.Linear) !usize {
         var yb: [3]u64 = undefined;
         for (set, 0..) |si, i| {
             const sh = shapes[si];
-            const f = fixed[si];
             const words = (sh.k / 16) * (sh.n / 16) * sh.halves * 4;
             const host = try r.a.alloc(u32, words);
             for (host) |*v| v.* = r.rng.int(u32);
@@ -245,7 +237,7 @@ fn groups(r: *Rig, lin: *const exl3.Linear) !usize {
             for (suh) |*v| v.* = @bitCast(@as(f16, @floatCast(r.rng.float(f32) * 2 - 1)));
             for (svh) |*v| v.* = @bitCast(@as(f16, @floatCast(r.rng.float(f32) * 2 - 1)));
             const st = exl3.strides(sh.k, sh.n, sh.halves, false);
-            ws[i] = .{ .trellis = try r.dev(std.mem.sliceAsBytes(host)), .suh = try r.dev(std.mem.sliceAsBytes(suh)), .svh = try r.dev(std.mem.sliceAsBytes(svh)), .k = @intCast(sh.k), .n = @intCast(sh.n), .halves = sh.halves, .codebook = .mul1, .stride_k = st.k, .stride_nb = st.nb, .sk = f.sk, .wk = f.wk };
+            ws[i] = .{ .trellis = try r.dev(std.mem.sliceAsBytes(host)), .suh = try r.dev(std.mem.sliceAsBytes(suh)), .svh = try r.dev(std.mem.sliceAsBytes(svh)), .k = @intCast(sh.k), .n = @intCast(sh.n), .halves = sh.halves, .codebook = .mul1, .stride_k = st.k, .stride_nb = st.nb };
             ya[i] = try r.zeros(16 * sh.n * 2);
             yb[i] = try r.zeros(16 * sh.n * 2);
         }
diff --git a/zig/tests/cuda/exl3_plan_f64.zig b/zig/tests/cuda/exl3_plan_f64.zig
deleted file mode 100644
index 6e8766c1cb..0000000000
--- a/zig/tests/cuda/exl3_plan_f64.zig
+++ /dev/null
@@ -1,97 +0,0 @@
-//! `tf-cuda-test exl3-plan-f64`: the 27B's EXL3 shapes against float64, under plan()'s launch and the fixed one.
-const std = @import("std");
-const cuda = @import("cuda");
-const check = @import("check.zig");
-const quant_ref_tests = @import("quant_ref_tests.zig");
-const exl3_tests = @import("exl3_tests.zig");
-const bench = @import("exl3_plan_bench.zig");
-const exl3fmt = @import("core").exl3_format;
-const exl3 = cuda.exl3;
-const Gpu = check.Gpu;
-const Rig = quant_ref_tests.RigT;
-
-/// One launch's error against float64: the largest share of the bound, the worst column and the root mean square.
-const Stats = struct { q: f64 = 0, worst: f64 = 0, col: usize = 0, rms: f64 = 0 };
-
-pub fn run(gpu: Gpu, only: ?[]const u8) !void {
-    var arena = std.heap.ArenaAllocator.init(gpu.gpa);
-    defer arena.deinit();
-    var s = try cuda.Stream.init(gpu.d, false);
-    defer s.deinit();
-    var prng = std.Random.DefaultPrng.init(13);
-    var r: Rig = .{ .gpu = gpu, .a = arena.allocator(), .s = s, .rng = prng.random() };
-    defer {
-        for (r.bufs.items) |*b| b.free();
-        r.bufs.deinit(gpu.gpa);
-    }
-    var lin = try exl3.Linear.load(gpu.d);
-    defer lin.unload();
-    var bad: usize = 0;
-    for (bench.shapes, bench.fixed) |sh, f| {
-        if (only) |o| if (!std.mem.eql(u8, o, sh.name)) continue;
-        bad += try shape(&r, &lin, sh, f.sk, f.wk);
-        for (r.bufs.items) |*b| b.free();
-        r.bufs.clearRetainingCapacity();
-    }
-    try check.expect(bad == 0, "exl3.plan.f64: {d} launches outside float64's bound", .{bad});
-    check.pass("exl3.plan.f64", .{});
-}
-
-/// Decode rows (16) and prompt rows (128) on identical inputs, each plan's fp32 out against one float64 reference.
-fn shape(r: *Rig, lin: *const exl3.Linear, sh: bench.Shape, sk: u32, wk: u32) !usize {
-    const width = exl3fmt.Width{ .halves = @intCast(sh.halves) };
-    const rows: usize = if (sh.n > 65536) 16 else 128; // the head scores decode rows and verify windows only
-    const trellis = try r.a.alloc(i16, (sh.k / 16) * (sh.n / 16) * width.tileWords());
-    for (trellis) |*v| v.* = r.rng.int(i16);
-    const suh = try r.a.alloc(u16, sh.k);
-    const svh = try r.a.alloc(u16, sh.n);
-    for (suh) |*v| v.* = @bitCast(@as(f16, @floatCast(r.rng.float(f32) * 2 - 1)));
-    for (svh) |*v| v.* = @bitCast(@as(f16, @floatCast(r.rng.float(f32) * 2 - 1)));
-    const x = try r.bfs(rows * sh.k, 1.0);
-    const ref = try exl3_tests.reference(r, width, .mul1, trellis, sh.k, sh.n, x, suh, svh, rows);
-    const st = exl3.strides(sh.k, sh.n, sh.halves, false);
-    const base: exl3.Weight = .{ .trellis = try r.dev(std.mem.sliceAsBytes(trellis)), .suh = try r.dev(std.mem.sliceAsBytes(suh)), .svh = try r.dev(std.mem.sliceAsBytes(svh)), .k = @intCast(sh.k), .n = @intCast(sh.n), .halves = sh.halves, .codebook = .mul1, .stride_k = st.k, .stride_nb = st.nb };
-    var tuned = base;
-    tuned.sk = sk;
-    tuned.wk = wk;
-    const dx = try r.dev(std.mem.sliceAsBytes(x));
-    const xh = try r.zeros(rows * sh.k * 2);
-    const z = try r.zeros(@max(exl3.Linear.zBytes(base, rows), exl3.Linear.zBytes(tuned, rows)) + 16);
-    const counters = try exl3_tests.zeroed(r, exl3.Linear.counterBytes(base));
-    const y = try r.zeros(rows * sh.n * 4);
-    var bad: usize = 0;
-    const ms: []const usize = if (rows == 16) &.{16} else &.{ 16, 128 };
-    for (ms) |m| {
-        var got: [2]Stats = undefined;
-        var outs: [2][]u8 = undefined;
-        for ([_]exl3.Weight{ base, tuned }, &got, &outs, [_][]const u8{ "plan()", "fixed" }) |w, *g, *o, name| {
-            try lin.matmul(r.s, w, dx, .bf16, m, y, .f32, xh, z, counters);
-            g.* = measure(try r.back(y, m * sh.n * 4), ref.want, ref.tol, m, sh.n);
-            if (g.q > 1) bad += 1;
-            try lin.matmul(r.s, w, dx, .bf16, m, y, .bf16, xh, z, counters);
-            o.* = try r.back(y, m * sh.n * 2);
-            const at = exl3.planOf(w);
-            std.debug.print("RESULT exl3.plan.f64 {s} {s} m={d} {s} sk={d} wk={d}: fp32 out worst {e:.3} (column {d}), rms {e:.3}, {d:.3} of the bound\n", .{ sh.name, if (m == 16) "decode" else "prompt", m, name, at.sk, at.wk, g.worst, g.col, g.rms, g.q });
-        }
-        var differ: usize = 0;
-        for (0..m * sh.n) |i| differ += @intFromBool(!std.mem.eql(u8, outs[0][2 * i ..][0..2], outs[1][2 * i ..][0..2]));
-        std.debug.print("RESULT exl3.plan.f64 {s} m={d}: fixed / plan() fp32 rms {d:.4}, worst {d:.4}; bf16 outputs that differ {d} of {d}\n", .{ sh.name, m, got[1].rms / got[0].rms, got[1].worst / got[0].worst, differ, m * sh.n });
-    }
-    return bad;
-}
-
-fn measure(raw: []const u8, want: []const f64, tol: []const f64, m: usize, n: usize) Stats {
-    var s: Stats = .{};
-    var sq: f64 = 0;
-    for (0..m * n) |i| {
-        const e = @abs(@as(f64, std.mem.bytesToValue(f32, raw[4 * i ..][0..4])) - want[i]);
-        s.q = @max(s.q, e / tol[i]);
-        sq += e * e;
-        if (e > s.worst) {
-            s.worst = e;
-            s.col = i % n;
-        }
-    }
-    s.rms = @sqrt(sq / @as(f64, @floatFromInt(m * n)));
-    return s;
-}
diff --git a/zig/tests/cuda/exl3_tests.zig b/zig/tests/cuda/exl3_tests.zig
index 0389f0863c..d16294c179 100644
--- a/zig/tests/cuda/exl3_tests.zig
+++ b/zig/tests/cuda/exl3_tests.zig
@@ -134,7 +134,7 @@ fn f64Linear(r: *Rig, lin: *const exl3.Linear, c: Cfg) !void {
     for (suh) |*v| v.* = @bitCast(@as(f16, @floatCast(r.rng.float(f32) * 2 - 1)));
     for (svh) |*v| v.* = @bitCast(@as(f16, @floatCast(r.rng.float(f32) * 2 - 1)));
     const x = try r.bfs(128 * k, 1.0);
-    const want = try reference(r, width, c.codebook, trellis, k, n, x, suh, svh, 128);
+    const want = try reference(r, width, c.codebook, trellis, k, n, x, suh, svh);
     const st = exl3.strides(k, n, c.halves, false);
     const w: exl3.Weight = .{
         .trellis = try r.dev(std.mem.sliceAsBytes(trellis)),
@@ -159,8 +159,9 @@ fn f64Linear(r: *Rig, lin: *const exl3.Linear, c: Cfg) !void {
     }
 }
 
-/// The Python engine's f64 reference for m (at most 128) rows: Wq fp16 from the trellis, fp16 xh, y, each |t| sum.
-pub fn reference(r: *Rig, width: exl3fmt.Width, cb: exl3fmt.Codebook, trellis: []const i16, k: usize, n: usize, x: []const u16, suh: []const u16, svh: []const u16, m: usize) !struct { want: []f64, tol: []f64 } {
+/// The Python engine's f64 reference: Wq fp16 from the trellis, fp16 xh, the y sums, and each element's |t| sum.
+fn reference(r: *Rig, width: exl3fmt.Width, cb: exl3fmt.Codebook, trellis: []const i16, k: usize, n: usize, x: []const u16, suh: []const u16, svh: []const u16) !struct { want: []f64, tol: []f64 } {
+    const m = 128;
     const wq = try r.a.alloc(f16, k * n);
     exl3fmt.unpack(width, cb, trellis, k / 16, n / 16, wq);
     const xs = try r.a.alloc(f64, m * k);
diff --git a/zig/tests/cuda/main.zig b/zig/tests/cuda/main.zig
index 3c37e4170b..2898ea8e06 100644
--- a/zig/tests/cuda/main.zig
+++ b/zig/tests/cuda/main.zig
@@ -42,7 +42,7 @@ const modelopt_tests = @import("modelopt_tests.zig");
 const exl3_tests = @import("exl3_tests.zig");
 const exl3_tp2_tests = @import("exl3_tp2_tests.zig");
 const exl3_plan_bench = @import("exl3_plan_bench.zig");
-const exl3_plan_f64 = @import("exl3_plan_f64.zig");
+const exl3_contract = @import("contract_exl3.zig");
 const exl3_candidate_tests = @import("exl3_candidate_tests.zig");
 const experts3_tests = @import("experts3_tests.zig");
 const moe_tests = @import("moe_tests.zig");
@@ -226,10 +226,10 @@ const usage =
     \\  moe-glue                  sigmoid top-k routing and the weighted combine against float64 references (synthetic)
     \\  dense-rows                bf16 rows times weight rows (routers, heads) against float64 references (synthetic)
     \\  exl3 [dir]                EXL3 decode-row linear: float64 checks, row invariance, the Python engine's fixture (dir none skips it)
-    \\  exl3-shapes [rows]        the 27B's EXL3 shapes once each with the fixed plans (1 and 16 rows), for ncu
+    \\  exl3-shapes [rows]        the 27B's EXL3 shapes once each at plan()'s launch (1 and 16 rows), for ncu
     \\  exl3-midm                 the mid-M kernels, linear_wc and the projection groups against the linear, byte for byte
     \\  exl3-plans [shape]        the 27B's EXL3 shapes: every (K splits, warps) and layout timed at 1 and 16 rows
-    \\  exl3-plan-f64 [shape]     the 27B's EXL3 shapes against float64 under plan()'s launch and the fixed one, 16 and 128 rows
+    \\  exl3-contract             EXL3's replacements of linear_body through gates 1-4, each control red (kernel-contract's subset)
     \\  exl3-tp2 <rank> <host:port> [dir]  EXL3 tensor parallel: rank shards vs the Python engine's, all-gathered (dir none skips them)
     \\  exl3-tiles [DIR]          EXL3's tile candidate: codebooks and tiles bit-equal, FP64, rows through qlinear and Dense
     \\  train-pack MODEL EXAMPLES [COUNT]   training passes holding several sequences against one a pass: differences, time
@@ -401,7 +401,7 @@ fn run(gpu: check.Gpu, cmd: []const u8, rest: []const [:0]const u8) !void {
     if (std.mem.eql(u8, cmd, "exl3-shapes")) return exl3_plan_bench.profileShapes(gpu, if (rest.len > 0) rest[0] else null);
     if (std.mem.eql(u8, cmd, "exl3-midm")) return exl3_plan_bench.midm(gpu);
     if (std.mem.eql(u8, cmd, "exl3-plans")) return exl3_plan_bench.run(gpu, if (rest.len > 0) rest[0] else null);
-    if (std.mem.eql(u8, cmd, "exl3-plan-f64")) return exl3_plan_f64.run(gpu, if (rest.len > 0) rest[0] else null);
+    if (std.mem.eql(u8, cmd, "exl3-contract")) return exl3_contract.run(gpu);
     if (std.mem.eql(u8, cmd, "exl3-tp2")) return exl3_tp2_tests.run(gpu, try arg(rest, 0), try arg(rest, 1), if (rest.len > 2) rest[2] else "none");
     if (std.mem.eql(u8, cmd, "sample")) return sample_tests.draws(gpu);
     if (std.mem.eql(u8, cmd, "topk")) return sample_tests.topk(gpu);

From 3ccb0d90ca38627c5d20d04f95aab3a7a46d2be1 Mon Sep 17 00:00:00 2001
From: BHCC2025 <241330250+BHCC2025@users.noreply.github.com>
Date: Wed, 28 Oct 2026 11:00:00 -0500
Subject: [PATCH 6/8] exl3 cuda: every mid-M, linear_wc and group form the
 dispatch reaches in the kernel contract

- contract_exl3 registers the mid-M kernels at 3 and 5 bits too, each with a control, and its served shapes reach
  every form plan() gives a width: both row passes, 4 and 8 warps, split and unsplit K, every linear_wc tile. The
  groups gain 4 warps unsplit.
- exl3.pick is matmul's dispatch alone. A host test sweeps K and N to 16384 through it and groupable, and fails on a
  form no registration's gate-3 shapes run. A family's fixed plan keeps the linear.
- Gate 4's stress weights are all-ones states, one weight at every width (0xaaaa decodes to two at 3 and 5 bits).
---
 docs/kernel-contract.md               |  11 +-
 zig/src/cuda/exl3.zig                 |  64 ++++++----
 zig/src/cuda/exl3_dense_test.zig      |  14 +++
 zig/tests/cuda/contract_arch_test.zig |   4 +
 zig/tests/cuda/contract_exl3.zig      | 165 ++++++++++++++++++++++----
 5 files changed, 208 insertions(+), 50 deletions(-)

diff --git a/docs/kernel-contract.md b/docs/kernel-contract.md
index ff170d9457..beda235dac 100644
--- a/docs/kernel-contract.md
+++ b/docs/kernel-contract.md
@@ -184,10 +184,13 @@ each row then L2-normed, against float64's ranges at their bf16 points and expf'
 CUDA also checks `dense_rows`' scaled epilogue (Dory's logits, fp32(bf16(x . w)) times a per-column scale), bit-equal to its
 unscaled tile's outputs rounded and scaled on the host.
 CUDA also checks EXL3's replacements of `linear_body` (`exl3_linear.cu`), each byte-equal to it at every row count it
-takes: the mid-M kernels and `linear_wc` (4- and 6-bit, 1-128 rows; `Linear.min_rows` 1 runs them below 17 rows), the
-projection groups against each member's own launch, and `tf_exl3_rot_group` against `tf_exl3_rot_in` (1-16 rows). Gate 4
-proves an fp16 sum would overflow on a constant-weight stress input. Four controls: another K split, swapped scales, a
-swapped suh.
+takes: the mid-M kernels (3, 4, 5 and 6 bits) and `linear_wc` (4 and 6 bits) at 1-128 rows (`Linear.min_rows` 1 runs
+them below 17 rows), the projection groups against each member's own launch, and `tf_exl3_rot_group` against
+`tf_exl3_rot_in` (1-16 rows). Their served shapes reach every form `plan()` gives them (row passes, warps, split or
+unsplit K, `linear_wc`'s tiles); a host test sweeps K and N to 16384 through `exl3.pick` and `groupable` and fails on a
+form no registration runs. A fixed plan keeps the linear. Gate 4 proves an fp16 sum would overflow on a constant-weight
+stress input (all-ones states: one weight at every width). Six controls: another K split (3, 4 and 5 bits; `linear_wc`),
+swapped scales, a swapped suh.
 HIP: the RMS norm rows, a wave a row (fp32, fp16 and bf16 out) and a block a row (fp16 and bf16 out).
 
 RADIO's fused bias, activation and residual (`tf_vit_bias_act`) replaces `tf_vit_bias` then
diff --git a/zig/src/cuda/exl3.zig b/zig/src/cuda/exl3.zig
index 86368fb17a..0c907654b4 100644
--- a/zig/src/cuda/exl3.zig
+++ b/zig/src/cuda/exl3.zig
@@ -115,6 +115,20 @@ pub fn variantIndex(halves: u32, codebook: Codebook, warps: u32) ?usize {
 /// From this many 128-column blocks a shape plans one K split, so a view that wide keeps the whole matrix's bits.
 pub const wide_blocks = 64;
 
+/// The kernel matmul launches after the rotation: a linear_wc, a mid-M kernel or the linear, by index.
+pub const Pick = union(enum) { wc: usize, mpg: usize, linear: usize };
+
+/// matmul's choice for m rows of `w` (null: no kernel for its width); a family's fixed plan keeps the linear.
+pub fn pick(w: Weight, m: usize, midm: bool, use_wc: bool, min_rows: usize) ?Pick {
+    const p = planOf(w);
+    const idx = variantIndex(w.halves, w.codebook, p.wk) orelse return null;
+    if (!midm or w.sk != 0) return .{ .linear = idx };
+    if (use_wc) if (wcPick(w, m, p.sk, p.wk, min_rows)) |wi| return .{ .wc = wi };
+    // 17-48 rows: 16-row groups; 49-128: 32-row groups, each group's tiles decoded once (linear.cu's dispatch)
+    if (m >= min_rows) if (mpgIndex(w.halves, w.codebook, p.wk, if (m > 48) 2 else 1)) |mi| return .{ .mpg = mi };
+    return .{ .linear = idx };
+}
+
 /// What one shape launches, from the shape alone so a row's bits never depend on the row count (linear.py plan).
 pub fn plan(k: usize, n: usize, blocks: usize, min_tiles: usize) struct { sk: u32, wk: u32 } {
     const kt = k / 16;
@@ -251,7 +265,7 @@ pub const Linear = struct {
         if (w.k % 128 != 0 or w.n % 128 != 0) return error.Exl3Shape;
         const p = planOf(w);
         if ((w.k / 16) % (p.sk * p.wk) != 0) return error.Exl3Shape;
-        const idx = variantIndex(w.halves, w.codebook, p.wk) orelse return error.UnsupportedExl3Width;
+        const how = pick(w, m, l.midm, l.use_wc, l.min_rows) orelse return error.UnsupportedExl3Width;
         if (p.sk > 1 and z == 0) return error.Exl3Scratch;
 
         var r: launch_.Args = .{};
@@ -283,32 +297,32 @@ pub const Linear = struct {
         args.add(@as(i32, @intCast(p.sk)));
         args.add(@as(i32, @intCast(m)));
         args.add(@as(i32, @intCast(w.k)));
-        // 17-48 rows: 16-row groups; 49-128: 32-row groups, each group's tiles decoded once (linear.cu's dispatch)
-        if (l.midm and l.use_wc) if (wcPick(w, m, p.sk, p.wk, l.min_rows)) |wi| {
-            const v = wc_variants[wi];
-            args.add(@as(i32, @intCast(p.wk))); // WcArgs: LinearArgs, then the plan's warps
-            const rg = 16 * v.mt;
-            return launch_.launch(l.wc[wi], .{
-                .grid = .{ .x = @intCast(w.n / 128 * ((m + rg - 1) / rg)), .y = if (v.fold) 1 else p.sk, .z = 1 },
-                .block = .{ .x = 256, .y = 1, .z = 1 },
-                .shared = wcShared(v),
-            }, s, &args);
-        };
-        const pg: u32 = if (m > 48) 2 else 1;
-        if (l.midm and m >= l.min_rows) if (mpgIndex(w.halves, w.codebook, p.wk, pg)) |mi| {
-            const groups = (m + 16 * pg - 1) / (16 * pg);
-            return launch_.launch(l.mpg[mi], .{
-                .grid = .{ .x = @intCast(w.n / 128 * groups), .y = p.sk, .z = 1 },
+        switch (how) {
+            .wc => |wi| {
+                const v = wc_variants[wi];
+                args.add(@as(i32, @intCast(p.wk))); // WcArgs: LinearArgs, then the plan's warps
+                const rg = 16 * v.mt;
+                try launch_.launch(l.wc[wi], .{
+                    .grid = .{ .x = @intCast(w.n / 128 * ((m + rg - 1) / rg)), .y = if (v.fold) 1 else p.sk, .z = 1 },
+                    .block = .{ .x = 256, .y = 1, .z = 1 },
+                    .shared = wcShared(v),
+                }, s, &args);
+            },
+            .mpg => |mi| {
+                const rg = 16 * mpg_variants[mi].p;
+                try launch_.launch(l.mpg[mi], .{
+                    .grid = .{ .x = @intCast(w.n / 128 * ((m + rg - 1) / rg)), .y = p.sk, .z = 1 },
+                    .block = .{ .x = @intCast(p.wk * 32), .y = 1, .z = 1 },
+                    .shared = sharedBytes(w, m),
+                }, s, &args);
+            },
+            .linear => |idx| try launch_.launch(l.fns[idx], .{
+                .grid = .{ .x = @intCast(w.n / 128 * p.sk), .y = 1, .z = 1 },
                 .block = .{ .x = @intCast(p.wk * 32), .y = 1, .z = 1 },
                 .shared = sharedBytes(w, m),
-            }, s, &args);
-        };
-        try launch_.launch(l.fns[idx], .{
-            .grid = .{ .x = @intCast(w.n / 128 * p.sk), .y = 1, .z = 1 },
-            .block = .{ .x = @intCast(p.wk * 32), .y = 1, .z = 1 },
-            .shared = sharedBytes(w, m),
-            .pdl = l.pdl, // prefetches its weights, then waits for the rotation
-        }, s, &args);
+                .pdl = l.pdl, // prefetches its weights, then waits for the rotation
+            }, s, &args),
+        }
     }
 
     /// y_i = x through each of `ws` in one rotation and one launch (a member's own bits); scratch back to back, zeroed.
diff --git a/zig/src/cuda/exl3_dense_test.zig b/zig/src/cuda/exl3_dense_test.zig
index 4e24837632..ca992a9b8a 100644
--- a/zig/src/cuda/exl3_dense_test.zig
+++ b/zig/src/cuda/exl3_dense_test.zig
@@ -84,3 +84,17 @@ test "after a dense layer's reserves, its gate/up group decodes 1-16 rows withou
     _ = try bare.groupScratch(s, &ws, 16);
     try std.testing.expect(mock.allocations > before);
 }
+
+test "pick: linear_wc, then the mid-M kernels from min_rows; the linear below them, for other widths and fixed plans" {
+    const w = weight(5120, 17408, 8);
+    try expectEqual(exl3.Pick{ .wc = 1 }, exl3.pick(w, 17, true, true, 17).?);
+    try std.testing.expect(exl3.pick(w, 36, true, true, 17).? == .mpg); // 33-40 rows, 4-bit: mid-M
+    try std.testing.expect(exl3.pick(w, 36, false, true, 17).? == .linear);
+    try std.testing.expect(exl3.pick(w, 16, true, true, 17).? == .linear);
+    try std.testing.expect(exl3.pick(weight(5120, 1024, 6), 64, true, true, 17).? == .mpg); // 3-bit
+    try std.testing.expect(exl3.pick(weight(5120, 1024, 7), 64, true, true, 17).? == .linear); // 3.5-bit: none
+    var fixed = w;
+    fixed.sk = 1;
+    fixed.wk = 8;
+    try std.testing.expect(exl3.pick(fixed, 64, true, true, 17).? == .linear); // (unregistered forms otherwise)
+}
diff --git a/zig/tests/cuda/contract_arch_test.zig b/zig/tests/cuda/contract_arch_test.zig
index f84022ccb2..b287eaf6b5 100644
--- a/zig/tests/cuda/contract_arch_test.zig
+++ b/zig/tests/cuda/contract_arch_test.zig
@@ -136,3 +136,7 @@ test "contract architecture: modern gate 3 retains bit equality and rejects a pl
         try std.testing.expectEqual(@as(usize, 0), cpu.references);
     }
 }
+
+test {
+    _ = @import("contract_exl3.zig"); // its coverage of the EXL3 dispatch's forms
+}
diff --git a/zig/tests/cuda/contract_exl3.zig b/zig/tests/cuda/contract_exl3.zig
index e83ce36e36..6b6a769b21 100644
--- a/zig/tests/cuda/contract_exl3.zig
+++ b/zig/tests/cuda/contract_exl3.zig
@@ -26,16 +26,49 @@ fn rotDepth(_: c.Shape) usize {
     return 7 + 8;
 }
 
-/// The engine's shapes (Qwen3.8-27B's, K 5120 / 6144): plan()'s splits from 1 to 8, 4 and 8 warps.
+/// The widths with mid-M kernels (exl3.mpg_variants); linear_wc takes 4- and 6-bit.
+const mid_widths = [_]u32{ 6, 8, 10, 12 };
+
+fn name(kind: Kind, halves: u32, control: bool) []const u8 {
+    inline for (mid_widths) |h| if (h == halves) return switch (kind) {
+        .midm => if (control) std.fmt.comptimePrint("control: exl3 {d}-bit mid-M kernels at another K split posing as linear_body", .{h / 2}) else std.fmt.comptimePrint("cuda exl3 mid-M kernels tf_exl3_mpg ({d}-bit, 1-128 rows)", .{h / 2}),
+        else => if (control) std.fmt.comptimePrint("control: exl3 {d}-bit linear_wc at another K split posing as linear_body", .{h / 2}) else std.fmt.comptimePrint("cuda exl3 linear_wc tf_exl3_wc ({d}-bit, 1-128 rows)", .{h / 2}),
+    };
+    unreachable;
+}
+
+fn sh(rows: usize, n: usize, k: usize) c.Shape {
+    return .{ .rows = rows, .n = n, .k = k };
+}
+
+// The 27B's shapes (K 5120 / 6144), then shapes reaching the rest of plan()'s forms (the host test below sweeps them)
+const qwen4 = [_]c.Shape{ sh(128, 1024, 5120), sh(100, 17408, 5120), sh(49, 6144, 5120), sh(33, 5120, 6144), sh(24, 12288, 5120), sh(64, 5120, 6144) };
+const qwen6 = [_]c.Shape{ sh(128, 2048, 5120), sh(97, 4096, 5120), sh(40, 2048, 5120) };
+const small_k = [_]c.Shape{ sh(17, 1024, 512), sh(49, 1024, 512), sh(17, 8192, 1024), sh(49, 8192, 1024) };
+const wc_k = [_]c.Shape{ sh(17, 1024, 640), sh(33, 1024, 512), sh(17, 1024, 1024), sh(17, 1024, 1408), sh(49, 1024, 640), sh(49, 1024, 1024), sh(49, 1024, 1408), sh(17, 4096, 1024), sh(17, 4096, 1280), sh(17, 8192, 1152), sh(33, 4096, 1280), sh(49, 8192, 1152) };
+const wc6_k = [_]c.Shape{ sh(17, 1024, 512), sh(17, 1024, 640), sh(33, 1024, 512), sh(17, 1024, 1024), sh(33, 1024, 640), sh(17, 1024, 1408), sh(33, 1024, 1024), sh(49, 1024, 1408), sh(17, 4096, 1024), sh(17, 4096, 1280), sh(33, 4096, 1024), sh(17, 8192, 1152), sh(49, 8192, 1152) };
+// n names the members (members())
+const groups = [_]c.Shape{ sh(16, 6144, 5120), sh(16, 17408, 5120), sh(7, 12288, 5120), sh(16, 9216, 512) };
+
+fn served(kind: Kind, halves: u32) []const c.Shape {
+    return switch (kind) {
+        .midm => switch (halves) {
+            8 => &(qwen4 ++ small_k[0..1].*),
+            12 => &(qwen6 ++ small_k[2..4].*),
+            else => &(qwen4[0..1].* ++ small_k),
+        },
+        .wc => if (halves == 8) &(qwen4 ++ wc_k) else &(qwen6 ++ wc6_k),
+        .group, .rot_group => &groups,
+    };
+}
+
+/// The served shapes: every form plan() gives the replacement at each width, every row edge at the first.
 pub fn spec(kind: Kind, halves: u32, control: bool) c.Spec {
     const rows = [3]c.Tile{ .{ .size = 16 }, .{}, .{} };
     const sums: c.Sums = .{ .f16 = 4, .bf16 = 0, .exact = false, .why = why };
     return switch (kind) {
         .midm, .wc => .{
-            .name = switch (kind) {
-                .midm => if (control) "control: exl3 mid-M kernels at another K split posing as linear_body" else if (halves == 8) "cuda exl3 mid-M kernels tf_exl3_mpg (4-bit, 1-128 rows)" else "cuda exl3 mid-M kernels tf_exl3_mpg (6-bit, 1-128 rows)",
-                else => if (control) "control: exl3 linear_wc at another K split posing as linear_body" else if (halves == 8) "cuda exl3 linear_wc tf_exl3_wc (4-bit, 1-128 rows)" else "cuda exl3 linear_wc tf_exl3_wc (6-bit, 1-128 rows)",
-            },
+            .name = name(kind, halves, control),
             .accum = .f32,
             .depth = depth,
             .ops = 4,
@@ -43,7 +76,7 @@ pub fn spec(kind: Kind, halves: u32, control: bool) c.Spec {
             .replaces = "tf_exl3_linear (linear_body)",
             .nan = .{ .stated = rule },
             .tiles = rows,
-            .served = if (halves == 8) &[_]c.Shape{ .{ .rows = 128, .n = 1024, .k = 5120 }, .{ .rows = 100, .n = 17408, .k = 5120 }, .{ .rows = 49, .n = 6144, .k = 5120 }, .{ .rows = 33, .n = 5120, .k = 6144 }, .{ .rows = 24, .n = 12288, .k = 5120 }, .{ .rows = 64, .n = 5120, .k = 6144 } } else &[_]c.Shape{ .{ .rows = 128, .n = 2048, .k = 5120 }, .{ .rows = 97, .n = 4096, .k = 5120 }, .{ .rows = 40, .n = 2048, .k = 5120 } },
+            .served = served(kind, halves),
             .sums = sums,
         },
         .group => .{
@@ -55,8 +88,7 @@ pub fn spec(kind: Kind, halves: u32, control: bool) c.Spec {
             .replaces = "each member's tf_exl3_linear (linear_body)",
             .nan = .{ .stated = rule },
             .tiles = rows,
-            // n names the members: 17408 gate + up, 12288 q + gate + qkv (8 warps), 6144 z + k_v (4 warps, K split)
-            .served = &[_]c.Shape{ .{ .rows = 16, .n = 6144, .k = 5120 }, .{ .rows = 16, .n = 17408, .k = 5120 }, .{ .rows = 7, .n = 12288, .k = 5120 } },
+            .served = served(kind, halves),
             .sums = sums,
         },
         .rot_group => .{
@@ -67,7 +99,7 @@ pub fn spec(kind: Kind, halves: u32, control: bool) c.Spec {
             .out = .f16,
             .replaces = "each member's tf_exl3_rot_in",
             .tiles = rows,
-            .served = &[_]c.Shape{ .{ .rows = 16, .n = 6144, .k = 5120 }, .{ .rows = 16, .n = 17408, .k = 5120 }, .{ .rows = 7, .n = 12288, .k = 5120 } },
+            .served = served(kind, halves),
             .sums = .{ .f16 = 4, .bf16 = 0, .exact = false, .why = "128-term rotations of bf16 inputs: fp32 is not exact on them, so gate 4 proves fp16's miss alone" },
         },
     };
@@ -86,9 +118,10 @@ const Mat = struct {
 /// The members a group shape names (n): their output widths.
 fn members(n: usize) []const usize {
     return switch (n) {
-        17408 => &.{ 17408, 17408 },
-        12288 => &.{ 12288, 17408, 10240 },
-        else => &.{ 6144, 1024 },
+        17408 => &.{ 17408, 17408 }, // gate + up, 8 warps
+        12288 => &.{ 12288, 17408, 10240 }, // q + gate + qkv, 8 warps
+        9216 => &.{ 8192, 1024 }, // two layers at K 512: 4 warps, no split
+        else => &.{ 6144, 1024 }, // z + k_v, 4 warps, K split
     };
 }
 
@@ -169,8 +202,8 @@ pub const Case = struct {
             m.trellis = try a.alloc(i16, (shape.k / 16) * (n / 16) * width.tileWords());
             m.suh = try a.alloc(u16, shape.k);
             m.svh = try a.alloc(u16, n);
-            if (fill == .stress and self.kind != .rot_group) { // one weight everywhere: a 2-bit period, scales 1
-                @memset(m.trellis, @bitCast(@as(u16, 0xaaaa)));
+            if (fill == .stress and self.kind != .rot_group) { // one weight at every width: all-ones states, scales 1
+                @memset(m.trellis, @bitCast(@as(u16, 0xffff)));
                 @memset(m.suh, 0x3c00);
                 @memset(m.svh, 0x3c00);
             } else {
@@ -404,19 +437,24 @@ pub const Case = struct {
 };
 
 /// Every registration and control, on one Linear the cases share; `add` appends them to the runner's list.
+const Reg = struct { k: Kind, h: u32 = 8, ctl: bool = false };
+
+/// The mid-M kernels at every width they have, linear_wc at 4 and 6 bits, the groups, rot_group, and the controls.
+pub const registrations = [_]Reg{
+    .{ .k = .midm, .h = 6 },              .{ .k = .midm },                   .{ .k = .midm, .h = 10 },              .{ .k = .midm, .h = 12 },
+    .{ .k = .wc },                        .{ .k = .wc, .h = 12 },            .{ .k = .group },                      .{ .k = .rot_group },
+    .{ .k = .midm, .h = 6, .ctl = true }, .{ .k = .midm, .ctl = true },      .{ .k = .midm, .h = 10, .ctl = true }, .{ .k = .wc, .ctl = true },
+    .{ .k = .group, .ctl = true },        .{ .k = .rot_group, .ctl = true },
+};
+
 pub const Set = struct {
     lin: ?exl3.Linear = null,
-    cases: [10]Case = undefined,
+    cases: [registrations.len]Case = undefined,
     n: usize = 0,
 
     pub fn add(self: *Set, gpu: check.Gpu, s: cuda.Stream, list: *std.ArrayList(g.Kernel)) !void {
         self.lin = try exl3.Linear.load(gpu.d);
-        const all = [_]struct { k: Kind, h: u32 = 8, ctl: bool = false }{
-            .{ .k = .midm },               .{ .k = .midm, .h = 12 },          .{ .k = .wc },                .{ .k = .wc, .h = 12 },
-            .{ .k = .group },              .{ .k = .rot_group },              .{ .k = .midm, .ctl = true }, .{ .k = .wc, .ctl = true },
-            .{ .k = .group, .ctl = true }, .{ .k = .rot_group, .ctl = true },
-        };
-        for (all) |x| {
+        for (registrations) |x| {
             self.cases[self.n] = .{ .rig = .{ .gpu = gpu, .s = s }, .lin = &self.lin.?, .kind = x.k, .halves = x.h, .control = x.ctl };
             self.n += 1;
         }
@@ -443,3 +481,88 @@ pub fn run(gpu: check.Gpu) !void {
     std.debug.print("SUMMARY exl3 {d} kernels: {d} pass gates 1-4, {d} fail; {d} controls: {d} red as planted, {d} missed\n", .{ t.kernels, t.passed, t.failed, t.controls, t.caught, t.missed });
     if (t.problems() > 0) return error.TestFailed;
 }
+
+/// A launch the dispatch can reach: its kernel (a group's by warps), and whether its plan splits K.
+const Form = struct { kind: enum { wc, mpg, group }, index: usize, split: bool };
+
+fn weightOf(k: usize, n: usize, halves: u32) exl3.Weight {
+    return .{ .trellis = 0, .suh = 0, .svh = 0, .k = @intCast(k), .n = @intCast(n), .halves = halves, .codebook = .mul1, .stride_k = 0, .stride_nb = 0 };
+}
+
+fn formOf(w: exl3.Weight, m: usize, use_wc: bool, min_rows: usize) ?Form {
+    const split = exl3.planOf(w).sk > 1;
+    return switch (exl3.pick(w, m, true, use_wc, min_rows) orelse return null) {
+        .wc => |i| .{ .kind = .wc, .index = i, .split = split },
+        .mpg => |i| .{ .kind = .mpg, .index = i, .split = split },
+        .linear => null,
+    };
+}
+
+fn put(a: std.mem.Allocator, set: *std.ArrayList(Form), f: Form) !void {
+    for (set.items) |x| if (std.meta.eql(x, f)) return;
+    try set.append(a, f);
+}
+
+/// Every form a registration's gate-3 shapes run (the mid-M and linear_wc cases at 1 row up, as Case.run sets them).
+fn covered(a: std.mem.Allocator) !std.ArrayList(Form) {
+    var set: std.ArrayList(Form) = .empty;
+    errdefer set.deinit(a);
+    for (registrations) |r| if (!r.ctl and r.k != .rot_group) {
+        const shapes = try c.shapes(a, spec(r.k, r.h, false));
+        defer a.free(shapes);
+        for (shapes) |s| if (r.k == .group) {
+            for (members(s.n)) |n| {
+                const w = weightOf(s.k, n, 8);
+                try put(a, &set, .{ .kind = .group, .index = exl3.planOf(w).wk, .split = exl3.planOf(w).sk > 1 });
+            }
+        } else if (formOf(weightOf(s.k, s.n, r.h), s.rows, r.k == .wc, 1)) |f| try put(a, &set, f);
+    };
+    return set;
+}
+
+// K and N in steps of 128 to 16384: past every threshold plan() and the dispatch read (64 blocks, 8 k tiles a warp)
+test "contract_exl3: every mid-M and linear_wc form plan() reaches at 17-128 rows is a registration's gate-3 shape" {
+    const a = std.testing.allocator;
+    var have = try covered(a);
+    defer have.deinit(a);
+    var widths: [8]u32 = undefined; // every width with a mid-M kernel or a linear_wc
+    var nw: usize = 0;
+    inline for (.{ exl3.mpg_variants, exl3.wc_variants }) |vs| for (vs) |v| if (std.mem.indexOfScalar(u32, widths[0..nw], v.halves) == null) {
+        widths[nw] = v.halves;
+        nw += 1;
+    };
+    var k: usize = 128;
+    while (k <= 16384) : (k += 128) {
+        var n: usize = 128;
+        while (n <= 16384) : (n += 128) for (widths[0..nw]) |h| for (17..129) |m| {
+            const f = formOf(weightOf(k, n, h), m, true, 17) orelse continue;
+            for (have.items) |x| {
+                if (std.meta.eql(x, f)) break;
+            } else {
+                std.debug.print("unregistered: {t} {d} split {} ({d}-bit, {d} rows, n {d}, k {d})\n", .{ f.kind, f.index, f.split, h / 2, m, n, k });
+                return error.TestUnexpectedResult;
+            }
+        };
+    }
+}
+
+test "contract_exl3: every group form groupable() admits is a registration's gate-3 shape" {
+    const a = std.testing.allocator;
+    var have = try covered(a);
+    defer have.deinit(a);
+    var k: usize = 128;
+    while (k <= 16384) : (k += 128) {
+        var n: usize = 128;
+        while (n <= 16384) : (n += 128) {
+            const w = weightOf(k, n, 8);
+            if (!exl3.groupable(&.{ w, w }, 16)) continue;
+            const f: Form = .{ .kind = .group, .index = exl3.planOf(w).wk, .split = exl3.planOf(w).sk > 1 };
+            for (have.items) |x| {
+                if (std.meta.eql(x, f)) break;
+            } else {
+                std.debug.print("unregistered group form: {d} warps, split {} (n {d}, k {d})\n", .{ f.index, f.split, n, k });
+                return error.TestUnexpectedResult;
+            }
+        }
+    }
+}

From 244bfddb3b0194f108867e779b0d39dec55c5b5f Mon Sep 17 00:00:00 2001
From: BHCC2025 <241330250+BHCC2025@users.noreply.github.com>
Date: Fri, 30 Oct 2026 21:11:23 -0500
Subject: [PATCH 7/8] exl3 cuda: the mid-M and linear_wc tables in
 exl3_midm.zig, so exl3.zig stays under 600 lines on main

#1475's grownCounters and its test took exl3.zig past the lean check's 600 lines; the variant tables, wcShared, wcPick and
mpgIndex move as they are (exl3.zig re-exports what other files name).
---
 zig/src/cuda/exl3.zig      | 58 ++++++--------------------------------
 zig/src/cuda/exl3_midm.zig | 56 ++++++++++++++++++++++++++++++++++++
 2 files changed, 64 insertions(+), 50 deletions(-)
 create mode 100644 zig/src/cuda/exl3_midm.zig

diff --git a/zig/src/cuda/exl3.zig b/zig/src/cuda/exl3.zig
index 0c907654b4..e98b7dd800 100644
--- a/zig/src/cuda/exl3.zig
+++ b/zig/src/cuda/exl3.zig
@@ -54,56 +54,14 @@ pub fn symbol(comptime halves: u32, comptime codebook: Codebook, comptime warps:
 
 pub const rot_in_symbol = "tf_exl3_rot_in";
 
-/// The mid-M kernels (17-128 rows, the same bits as the linear's): mul1 at 3 to 6 bits, 2, 4 and 8 warps, P 1 and 2.
-pub const Mpg = struct { halves: u32, warps: u32, p: u32 };
-pub const mpg_variants = blk: {
-    var vs: [24]Mpg = undefined;
-    var i: usize = 0;
-    for ([_]u32{ 6, 8, 10, 12 }) |h| for ([_]u32{ 1, 2 }) |p| for ([_]u32{ 2, 4, 8 }) |w| {
-        vs[i] = .{ .halves = h, .warps = w, .p = p };
-        i += 1;
-    };
-    break :blk vs;
-};
-
-/// linear_wc (17-128 rows, 4- and 6-bit mul1): 16 mt rows a program, ks k steps a stage, fold: every split in one.
-pub const Wc = struct { halves: u32, mt: u32, ks: u32, fold: bool };
-pub const wc_variants = blk: {
-    var vs: [16]Wc = undefined;
-    var i: usize = 0;
-    for ([_]u32{ 8, 12 }) |h| for ([_]bool{ false, true }) |f| for ([_]u32{ 2, 4 }) |mt| for ([_]u32{ 2, 4 }) |ks| {
-        vs[i] = .{ .halves = h, .mt = mt, .ks = ks, .fold = f };
-        i += 1;
-    };
-    break :blk vs;
-};
-
-fn wcShared(v: Wc) u32 {
-    const raw = 8 * 4 * v.halves;
-    const stage = v.ks * raw * 4 + v.ks * 16 * v.mt * 32;
-    return @max(4 * stage, 16 * v.mt * 128 * 4);
-}
-
-/// linear.cu dispatch_wc: which linear_wc takes m rows of this weight, or null for the mid-M kernels.
-pub fn wcPick(w: Weight, m: usize, sk: u32, wk: u32, min_rows: usize) ?usize {
-    if (w.codebook != .mul1 or (w.halves != 8 and w.halves != 12)) return null;
-    const per_warp = w.k / 16 / sk / wk;
-    if (per_warp % 2 != 0 or m < min_rows or m > 128) return null;
-    const nb = w.n / 128;
-    const fold = sk > 1 and nb >= 32;
-    if (!fold and w.halves == 8 and m > 32 and m <= 40) return null;
-    const four = per_warp % 4 == 0 and !(w.halves == 12 and m > 96);
-    const mt2 = m <= 32 or (m > 64 and m <= 96) or (fold and m > 96 and nb >= 64);
-    const want: Wc = .{ .halves = w.halves, .mt = if (mt2) 2 else 4, .ks = if (four) 4 else 2, .fold = fold };
-    for (wc_variants, 0..) |v, i| if (std.meta.eql(v, want)) return i;
-    return null;
-}
-
-fn mpgIndex(halves: u32, codebook: Codebook, warps: u32, p: u32) ?usize {
-    if (codebook != .mul1) return null;
-    for (mpg_variants, 0..) |v, i| if (v.halves == halves and v.warps == warps and v.p == p) return i;
-    return null;
-}
+const midm_tables = @import("exl3_midm.zig");
+pub const Mpg = midm_tables.Mpg;
+pub const mpg_variants = midm_tables.mpg_variants;
+pub const Wc = midm_tables.Wc;
+pub const wc_variants = midm_tables.wc_variants;
+pub const wcPick = midm_tables.wcPick;
+const wcShared = midm_tables.wcShared;
+const mpgIndex = midm_tables.mpgIndex;
 
 pub fn variantIndex(halves: u32, codebook: Codebook, warps: u32) ?usize {
     for (variants, 0..) |v, i| {
diff --git a/zig/src/cuda/exl3_midm.zig b/zig/src/cuda/exl3_midm.zig
new file mode 100644
index 0000000000..6ed72e2fa9
--- /dev/null
+++ b/zig/src/cuda/exl3_midm.zig
@@ -0,0 +1,56 @@
+//! The EXL3 mid-M kernels and linear_wc (17-128 rows): their variants, shared bytes and which one takes a weight.
+const std = @import("std");
+const exl3 = @import("exl3.zig");
+const Weight = exl3.Weight;
+const Codebook = exl3.Codebook;
+
+/// The mid-M kernels (17-128 rows, the same bits as the linear's): mul1 at 3 to 6 bits, 2, 4 and 8 warps, P 1 and 2.
+pub const Mpg = struct { halves: u32, warps: u32, p: u32 };
+pub const mpg_variants = blk: {
+    var vs: [24]Mpg = undefined;
+    var i: usize = 0;
+    for ([_]u32{ 6, 8, 10, 12 }) |h| for ([_]u32{ 1, 2 }) |p| for ([_]u32{ 2, 4, 8 }) |w| {
+        vs[i] = .{ .halves = h, .warps = w, .p = p };
+        i += 1;
+    };
+    break :blk vs;
+};
+
+/// linear_wc (17-128 rows, 4- and 6-bit mul1): 16 mt rows a program, ks k steps a stage, fold: every split in one.
+pub const Wc = struct { halves: u32, mt: u32, ks: u32, fold: bool };
+pub const wc_variants = blk: {
+    var vs: [16]Wc = undefined;
+    var i: usize = 0;
+    for ([_]u32{ 8, 12 }) |h| for ([_]bool{ false, true }) |f| for ([_]u32{ 2, 4 }) |mt| for ([_]u32{ 2, 4 }) |ks| {
+        vs[i] = .{ .halves = h, .mt = mt, .ks = ks, .fold = f };
+        i += 1;
+    };
+    break :blk vs;
+};
+
+pub fn wcShared(v: Wc) u32 {
+    const raw = 8 * 4 * v.halves;
+    const stage = v.ks * raw * 4 + v.ks * 16 * v.mt * 32;
+    return @max(4 * stage, 16 * v.mt * 128 * 4);
+}
+
+/// linear.cu dispatch_wc: which linear_wc takes m rows of this weight, or null for the mid-M kernels.
+pub fn wcPick(w: Weight, m: usize, sk: u32, wk: u32, min_rows: usize) ?usize {
+    if (w.codebook != .mul1 or (w.halves != 8 and w.halves != 12)) return null;
+    const per_warp = w.k / 16 / sk / wk;
+    if (per_warp % 2 != 0 or m < min_rows or m > 128) return null;
+    const nb = w.n / 128;
+    const fold = sk > 1 and nb >= 32;
+    if (!fold and w.halves == 8 and m > 32 and m <= 40) return null;
+    const four = per_warp % 4 == 0 and !(w.halves == 12 and m > 96);
+    const mt2 = m <= 32 or (m > 64 and m <= 96) or (fold and m > 96 and nb >= 64);
+    const want: Wc = .{ .halves = w.halves, .mt = if (mt2) 2 else 4, .ks = if (four) 4 else 2, .fold = fold };
+    for (wc_variants, 0..) |v, i| if (std.meta.eql(v, want)) return i;
+    return null;
+}
+
+pub fn mpgIndex(halves: u32, codebook: Codebook, warps: u32, p: u32) ?usize {
+    if (codebook != .mul1) return null;
+    for (mpg_variants, 0..) |v, i| if (v.halves == halves and v.warps == warps and v.p == p) return i;
+    return null;
+}

From c7e04357497251245fb5f14f7ea8209ad646bed2 Mon Sep 17 00:00:00 2001
From: BHCC2025 <241330250+BHCC2025@users.noreply.github.com>
Date: Sat, 31 Oct 2026 09:31:17 -0500
Subject: [PATCH 8/8] kernel contract: the EXL3 set joins the family sets, so
 contract_tests.zig stays under 600 lines

---
 zig/tests/cuda/contract_family_sets.zig | 16 ++++++++++++++--
 zig/tests/cuda/contract_tests.zig       |  4 ----
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/zig/tests/cuda/contract_family_sets.zig b/zig/tests/cuda/contract_family_sets.zig
index 62f5315456..3631c95370 100644
--- a/zig/tests/cuda/contract_family_sets.zig
+++ b/zig/tests/cuda/contract_family_sets.zig
@@ -1,4 +1,4 @@
-//! The family sets in the full kernel contract: Nemotron's pocket, MiniMax-M3's.
+//! The family sets in the full kernel contract: Nemotron's pocket, MiniMax-M3's, the dense EXL3 projections'.
 const std = @import("std");
 const cuda = @import("cuda");
 const check = @import("check.zig");
@@ -7,16 +7,28 @@ const g = @import("core").contract.gates;
 pub const Sets = struct {
     pocket: *@import("contract_pocket.zig").Set,
     minimax: *@import("contract_minimax.zig").Set,
+    exl3: *@import("contract_exl3.zig").Set,
+    a: std.mem.Allocator,
 
     /// Each set's kernels appended to `list`, in this order.
     pub fn init(gpu: check.Gpu, s: cuda.Stream, a: std.mem.Allocator, list: *std.ArrayList(g.Kernel)) !Sets {
         const pocket = try @import("contract_pocket.zig").Set.init(gpu, s, a, list);
         errdefer pocket.deinit();
         const minimax = try @import("contract_minimax.zig").Set.init(gpu, s, a, list);
-        return .{ .pocket = pocket, .minimax = minimax };
+        errdefer minimax.deinit();
+        const exl3 = try a.create(@import("contract_exl3.zig").Set);
+        exl3.* = .{};
+        errdefer {
+            exl3.deinit();
+            a.destroy(exl3);
+        }
+        try exl3.add(gpu, s, list);
+        return .{ .pocket = pocket, .minimax = minimax, .exl3 = exl3, .a = a };
     }
 
     pub fn deinit(sets: Sets) void {
+        sets.exl3.deinit();
+        sets.a.destroy(sets.exl3);
         sets.minimax.deinit();
         sets.pocket.deinit();
     }
diff --git a/zig/tests/cuda/contract_tests.zig b/zig/tests/cuda/contract_tests.zig
index f641dfbf15..812cd3dde4 100644
--- a/zig/tests/cuda/contract_tests.zig
+++ b/zig/tests/cuda/contract_tests.zig
@@ -9,7 +9,6 @@ const rows_k = @import("contract_rows.zig");
 const tail_k = @import("contract_norms_tail.zig");
 const tiles_k = @import("contract_rms_tiles.zig");
 const dory_k = @import("contract_dory.zig");
-const exl3_k = @import("contract_exl3.zig");
 const router_k = @import("contract_router.zig");
 const mm_k = @import("contract_mm.zig");
 const prefill_many_k = @import("contract_qmm_prefill_many.zig");
@@ -160,9 +159,6 @@ pub fn run(gpu: check.Gpu) !void {
     var dory: dory_k.Dory = .{};
     defer dory.deinit();
     try dory.add(gpu, s, &norms, mods[5], &kernels);
-    var exl3_set: exl3_k.Set = .{};
-    defer exl3_set.deinit();
-    try exl3_set.add(gpu, s, &kernels);
     var logits = router_k.Logits{ .rig = .{ .gpu = gpu, .s = s }, .f = try mods[0].function("tf_nemo_router") };
     defer logits.deinit();
     var logits_tiles = router_k.Logits{ .rig = .{ .gpu = gpu, .s = s }, .f = logits.f, .tiles = try mods[0].function("tf_nemo_router_tiles") };