From 3d65c90d04d337e88f2b1f7f0061f40a5324e662 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Thu, 8 Oct 2026 20:19:32 -0600 Subject: [PATCH 001/431] sycl : Q5_K reorder-layout MMVQ and fused GLU (#29375) --- ggml/src/ggml-sycl/ggml-sycl.cpp | 13 +++- ggml/src/ggml-sycl/mmvq.cpp | 108 +++++++++++++++++-------------- ggml/src/ggml-sycl/mmvq.hpp | 6 ++ ggml/src/ggml-sycl/vecdotq.hpp | 90 ++++++++++++++++---------- 4 files changed, 133 insertions(+), 84 deletions(-) diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 2a5184e01283..d2c1a1a44839 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -5102,12 +5102,19 @@ static bool ggml_sycl_mul_mat_glu_mmvq_fused(ggml_backend_sycl_context & ctx, gg return false; } - // quant pairs the reorder kernel cannot serve (mixed gate/up types) take the - // standard-layout fused path instead; q4_K keeps the reorder path below - if (wg->type != GGML_TYPE_Q4_K || wu->type != GGML_TYPE_Q4_K) { + // quant pairs the reorder kernel does not serve (mixed gate/up types, q5_K off BMG) take the + // standard-layout fused path instead; same-type q4_K / q5_K keep the reorder path below + const bool reorder_pair = wg->type == wu->type && + (wu->type == GGML_TYPE_Q4_K || (wu->type == GGML_TYPE_Q5_K && ggml_sycl_q5_k_mmvq_reuse(ctx.device))); + if (!reorder_pair) { return ggml_sycl_mul_mat_glu_mmvq_plain(ctx, glu, gate, up, wu, wg, act); } + // past 5 columns the two unfused q5_K GEMVs are faster than the fused kernel + if (wu->type == GGML_TYPE_Q5_K && act->ne[1] > 5) { + return false; + } + // install the reorder (SoA) layout the fused kernel needs, as the unfused mmvq path would; // a no-op once done. after the bail checks so a declined op does not pay for it. opt_for_reorder(&ctx, wu, act, up, mul_mat_algo::MMVQ); diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cpp index ceaacb59ece5..e8b293a97d75 100644 --- a/ggml/src/ggml-sycl/mmvq.cpp +++ b/ggml/src/ggml-sycl/mmvq.cpp @@ -110,7 +110,8 @@ static void mul_mat_vec_q_reorder(const void * __restrict__ vx, const void * __r // With has_fusion, `vgate` is a second weight matrix sharing vx's shape, stride and reorder // layout: one pass computes both row dot products and the epilogue writes glu(gate, up). -template +template ::value> static void mul_mat_vec_q_reorder_ncols(const void * __restrict__ vx, const void * __restrict__ vgate, const void * __restrict__ vy, float * __restrict__ dst, const int ncols, const int nrows, const int stride_col_y_bytes, const int stride_col_dst, @@ -181,7 +182,7 @@ static void mul_mat_vec_q_reorder_ncols(const void * __restrict__ vx, const void } } } - } else if constexpr (reorder_vec_dot_shared_weights::value) { + } else if constexpr (shared_weights) { const int ibx = row0 * blocks_per_row + i; const auto bx_offset = block_type::get_block_offset(ibx, nblocks); const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx); @@ -1945,8 +1946,8 @@ static void reorder_mul_mat_vec_q5_k_q8_1_sycl(const void * vx, const void * vy, }); } -template -static void reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols( +template +static void reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols_impl( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int stride_col_y_bytes, const int stride_col_dst, @@ -1954,20 +1955,35 @@ static void reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols( GGML_ASSERT(ncols % QK_K == 0); constexpr size_t num_subgroups = WARP_SIZE; - const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups * rows_per_sg); const sycl::range<3> block_nums(1, 1, block_num_y); const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); stream->submit([&](sycl::handler & cgh) { cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_reorder_ncols, ncols_dst>( + mul_mat_vec_q_reorder_ncols, ncols_dst, + /*has_fusion=*/ false, rows_per_sg, shared_weights>( vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item); }); }); } +template +static void reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + if (ggml_sycl_q5_k_mmvq_reuse(ggml_sycl_get_device())) { + constexpr int rows_per_sg = ncols_dst >= 3 ? 2 : 1; + reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols_impl(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); + } else { + reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols_impl(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); + } +} + static void reorder_mul_mat_vec_q5_k_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int ncols_dst, @@ -3129,8 +3145,11 @@ static void launch_mul_mat_vec_q_reorder_glu(const void * vx, const void * vgate const int ncols, const int nrows, const int stride_col_y_bytes, const int stride_col_dst, const ggml_glu_op glu_op, dpct::queue_ptr stream) { + // q4_K pairs rows for 3..4 columns, q5_K for 3..5 + constexpr int row_pair_max = reorder_vec_dot_q_sycl::gtype == GGML_TYPE_Q5_K ? 5 : 4; constexpr int rows_per_sg = - reorder_vec_dot_shared_activations::value && ncols_dst >= 3 && ncols_dst <= 4 + reorder_vec_dot_shared_activations::value && ncols_dst >= 3 && + ncols_dst <= row_pair_max ? 2 : 1; launch_mul_mat_vec_q_reorder_glu_impl(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, stream); @@ -3321,55 +3340,48 @@ bool ggml_sycl_mul_mat_vec_q_glu_plain(enum ggml_type gate_type, enum ggml_type return false; } +template +static bool mul_mat_vec_q_glu_reorder_ncols(enum ggml_glu_op glu_op, const void * vx, const void * vgate, + const void * vy, float * dst, int ncols, int nrows, int ncols_dst, + int stride_col_y_bytes, int stride_col_dst, dpct::queue_ptr stream) { + using vec_dot = reorder_vec_dot_q_sycl; + + auto launch = [&](auto I) -> bool { + constexpr int n = decltype(I)::value; + if (ncols_dst != n) { + return false; + } + if constexpr (type == GGML_TYPE_Q4_K && n == 2) { + if (nrows >= Q4_K_MMVQ_ROW_PAIR_MIN_NROWS) { + launch_mul_mat_vec_q_reorder_glu_impl(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, stream); + return true; + } + } + launch_mul_mat_vec_q_reorder_glu(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, + stride_col_dst, glu_op, stream); + return true; + }; + + // unary fold over launch + return (launch(std::integral_constant{}) || ...); +} + bool ggml_sycl_mul_mat_vec_q_glu_reorder(enum ggml_type src0_type, enum ggml_glu_op glu_op, const void * vx, const void * vgate, const void * vy, float * dst, int ncols, int nrows, int ncols_dst, int stride_col_y_bytes, int stride_col_dst, dpct::queue_ptr stream) { - if (src0_type != GGML_TYPE_Q4_K) { - return false; - } if (glu_op != GGML_GLU_OP_SWIGLU && glu_op != GGML_GLU_OP_GEGLU) { return false; } - using vec_dot = reorder_vec_dot_q_sycl; - - switch (ncols_dst) { - case 1: - launch_mul_mat_vec_q_reorder_glu(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, - stride_col_dst, glu_op, stream); - return true; - case 2: - if (nrows >= Q4_K_MMVQ_ROW_PAIR_MIN_NROWS) { - launch_mul_mat_vec_q_reorder_glu_impl(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, stream); - } else { - launch_mul_mat_vec_q_reorder_glu_impl(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, stream); - } - return true; - case 3: - launch_mul_mat_vec_q_reorder_glu(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, - stride_col_dst, glu_op, stream); - return true; - case 4: - launch_mul_mat_vec_q_reorder_glu(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, - stride_col_dst, glu_op, stream); - return true; - case 5: - launch_mul_mat_vec_q_reorder_glu(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, - stride_col_dst, glu_op, stream); - return true; - case 6: - launch_mul_mat_vec_q_reorder_glu(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, - stride_col_dst, glu_op, stream); - return true; - case 7: - launch_mul_mat_vec_q_reorder_glu(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, - stride_col_dst, glu_op, stream); - return true; - case 8: - launch_mul_mat_vec_q_reorder_glu(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, - stride_col_dst, glu_op, stream); - return true; + switch (src0_type) { + case GGML_TYPE_Q4_K: + return mul_mat_vec_q_glu_reorder_ncols( + glu_op, vx, vgate, vy, dst, ncols, nrows, ncols_dst, stride_col_y_bytes, stride_col_dst, stream); + case GGML_TYPE_Q5_K: + // fusion declines q5_K past 5 columns + return mul_mat_vec_q_glu_reorder_ncols( + glu_op, vx, vgate, vy, dst, ncols, nrows, ncols_dst, stride_col_y_bytes, stride_col_dst, stream); default: return false; } diff --git a/ggml/src/ggml-sycl/mmvq.hpp b/ggml/src/ggml-sycl/mmvq.hpp index 7fb9cf6f82e6..3a0ef07d67e4 100644 --- a/ggml/src/ggml-sycl/mmvq.hpp +++ b/ggml/src/ggml-sycl/mmvq.hpp @@ -15,6 +15,12 @@ #include "common.hpp" +// q5_K multi-column MMVQ shares weights across columns, pairs rows and fuses gate/up in the reorder +// layout: faster on Xe2 (BMG), so untested archs keep the per-column kernel +inline bool ggml_sycl_q5_k_mmvq_reuse(int device) { + const gpu_arch arch = ggml_sycl_info().devices[device].hw_info.arch; + return arch == gpu_arch::intel_gpu_bmg_g21 || arch == gpu_arch::intel_gpu_bmg_g31; +} void ggml_sycl_op_mul_mat_vec_q( ggml_backend_sycl_context & ctx, diff --git a/ggml/src/ggml-sycl/vecdotq.hpp b/ggml/src/ggml-sycl/vecdotq.hpp index cc6ae6a8d3df..6ae951525e87 100644 --- a/ggml/src/ggml-sycl/vecdotq.hpp +++ b/ggml/src/ggml-sycl/vecdotq.hpp @@ -362,6 +362,10 @@ template <> struct reorder_vec_dot_shared_weights { static constexpr bool value = true; }; +template <> struct reorder_vec_dot_shared_weights { + static constexpr bool value = true; +}; + template struct reorder_vec_dot_shared_activations { static constexpr bool value = false; }; @@ -370,6 +374,10 @@ template <> struct reorder_vec_dot_shared_activations { static constexpr bool value = true; }; +template <> struct reorder_vec_dot_shared_activations { + static constexpr bool value = true; +}; + template <> struct reorder_vec_dot_q_sycl { static constexpr ggml_type gtype = GGML_TYPE_Q4_0; @@ -676,56 +684,72 @@ template <> struct reorder_vec_dot_q_sycl { using q5_k_block = ggml_sycl_reordered::block_q_t; using q5_k_traits = typename q5_k_block::traits; - __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair ibx_offset, - const std::pair d_offset, const int8_t * q8_1_quant_ptr, - const sycl::half2 * q8_1_ds, const int & iqs) { - const uint8_t * base = static_cast(vbq); - const uint8_t * qs = base + ibx_offset.first; // low 4 bits - const uint8_t * qh_base = base + ibx_offset.second; // high bit - const uint8_t * scs = base + d_offset.first; - const ggml_half2 * dms = reinterpret_cast(base + d_offset.second); + struct weights { + int vl[2]; + int vh[2]; + uint16_t aux[2]; + ggml_half2 dm; + }; + + // same activation layout as Q4_K + static_assert(QR5_K == QR4_K); + using activations = reorder_vec_dot_q_sycl::activations; + + __dpct_inline__ static weights load(const void * __restrict__ vbq, const std::pair ibx_offset, + const std::pair d_offset, const int & iqs) { + const uint8_t * base = static_cast(vbq); + const uint8_t * qs = base + ibx_offset.first; // low 4 bits + const uint8_t * qh_base = base + ibx_offset.second; // high bit + const uint8_t * scs = base + d_offset.first; + const ggml_half2 * dms = reinterpret_cast(base + d_offset.second); const int bq8_offset = QR5_K * ((iqs / 2) / (QI8_1 / 2)); const int * ql_ptr = (const int *) (qs + 16 * bq8_offset + 4 * ((iqs / 2) % 4)); const int * qh_ptr = (const int *) (qh_base + 4 * ((iqs / 2) % 4)); const uint16_t * scales = (const uint16_t *) scs; - int vl[2]; - int vh[2]; - int u[2 * QR5_K]; - float d8[QR5_K]; - - vl[0] = ql_ptr[0]; - vl[1] = ql_ptr[4]; + weights w; + w.vl[0] = ql_ptr[0]; + w.vl[1] = ql_ptr[4]; - vh[0] = qh_ptr[0] >> bq8_offset; - vh[1] = qh_ptr[4] >> bq8_offset; + w.vh[0] = qh_ptr[0] >> bq8_offset; + w.vh[1] = qh_ptr[4] >> bq8_offset; - uint16_t aux[2]; const int j = (QR5_K * ((iqs / 2) / (QI8_1 / 2))) / 2; if (j < 2) { - aux[0] = scales[j + 0] & 0x3f3f; - aux[1] = scales[j + 2] & 0x3f3f; + w.aux[0] = scales[j + 0] & 0x3f3f; + w.aux[1] = scales[j + 2] & 0x3f3f; } else { - aux[0] = ((scales[j + 2] >> 0) & 0x0f0f) | ((scales[j - 2] & 0xc0c0) >> 2); - aux[1] = ((scales[j + 2] >> 4) & 0x0f0f) | ((scales[j - 0] & 0xc0c0) >> 2); + w.aux[0] = ((scales[j + 2] >> 0) & 0x0f0f) | ((scales[j - 2] & 0xc0c0) >> 2); + w.aux[1] = ((scales[j + 2] >> 4) & 0x0f0f) | ((scales[j - 0] & 0xc0c0) >> 2); } - const uint8_t * sc = (const uint8_t *) aux; - const uint8_t * m = sc + 2; + w.dm = *dms; - for (int i = 0; i < QR5_K; ++i) { - const int8_t* quant_base_ptr = q8_1_quant_ptr + (bq8_offset + i) * QK8_1; - sycl::half2 ds_values = *(q8_1_ds + bq8_offset + i); + return w; + } - d8[i] = ds_values[0]; + __dpct_inline__ static activations load_activations(const int8_t * q8_1_quant_ptr, + const sycl::half2 * q8_1_ds, const int & iqs) { + return reorder_vec_dot_q_sycl::load_activations(q8_1_quant_ptr, q8_1_ds, iqs); + } - const int * q8 = (const int *) quant_base_ptr + ((iqs / 2) % 4); - u[2 * i + 0] = q8[0]; - u[2 * i + 1] = q8[4]; - } + __dpct_inline__ static float apply(const weights & w, const activations & a) { + const uint8_t * sc = (const uint8_t *) w.aux; + const uint8_t * m = sc + 2; + + return vec_dot_q5_K_q8_1_impl_vmmq(w.vl, w.vh, a.u, sc, m, w.dm, a.d8); + } - return vec_dot_q5_K_q8_1_impl_vmmq(vl, vh, u, sc, m, *dms, d8); + __dpct_inline__ static float dot(const weights & w, const int8_t * q8_1_quant_ptr, + const sycl::half2 * q8_1_ds, const int & iqs) { + return apply(w, load_activations(q8_1_quant_ptr, q8_1_ds, iqs)); + } + + __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair ibx_offset, + const std::pair d_offset, const int8_t * q8_1_quant_ptr, + const sycl::half2 * q8_1_ds, const int & iqs) { + return dot(load(vbq, ibx_offset, d_offset, iqs), q8_1_quant_ptr, q8_1_ds, iqs); } }; From d4d82d67f48bb8f51e9a3518063e49d9ee54f58b Mon Sep 17 00:00:00 2001 From: Mark Todorovich Date: Thu, 8 Oct 2026 23:04:09 -0700 Subject: [PATCH 002/431] hexagon: fix IM2COL patch-embed DMA ring overflow (#30189) * hexagon: fix IM2COL patch-embed DMA ring overflow The exact-tiling (stride == kernel, no pad/dilation) IM2COL DMA kernel issues IC*KH DDR->VTCM descriptors per output row without checking the return value of dma_queue_push(), and then pops IC*KH times. The per-thread DMA ring holds 256 entries and a push into a full ring returns false and drops the transfer, so for IC*KH > 255 the remaining rows of the VTCM staging buffer were never written and stale data (often NaN/inf) leaked into the output. Solution is to retire the oldest descriptor when the ring is full, just as the blocked kernel in the same file already does, and wait with dma_queue_flush(). For testing, added exact-tiling test cases with IC*KH > 256 (2D 1x1, 2D 2x2 patch embed and 1D, F16 and F32 dst), which fail on HTP without this fix. * Apply suggestion from @max-krasnyansky --------- Co-authored-by: Max Krasnyansky --- ggml/src/ggml-hexagon/htp/im2col-ops.c | 11 +++++++---- tests/test-backend-ops.cpp | 8 ++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/im2col-ops.c b/ggml/src/ggml-hexagon/htp/im2col-ops.c index 2e05cf3e1002..b99d711c964d 100644 --- a/ggml/src/ggml-hexagon/htp/im2col-ops.c +++ b/ggml/src/ggml-hexagon/htp/im2col-ops.c @@ -351,12 +351,15 @@ IM2COL_BLOCKED_DMA_BODY(im2col_blocked_dma_f32_thread, float, hvx_copy_f32_uu, const dma_addr_t vsrc = ok \ ? (src_data + (size_t) ((in * IC + iic) * IH + iih) * IW * sizeof(float)) \ : src_data; \ - dma_queue_push(dma_q, dma_make_data(vdst, vsrc), \ - IW * sizeof(float), IW * sizeof(float), IW * sizeof(float), ok ? 1 : 0); \ + /* IC*KH descriptors per row can exceed the ring capacity: retire the oldest when full */ \ + while (!dma_queue_push(dma_q, dma_make_data(vdst, vsrc), \ + IW * sizeof(float), IW * sizeof(float), IW * sizeof(float), \ + ok ? 1 : 0)) { \ + dma_queue_pop(dma_q); \ + } \ } \ } \ - for (uint32_t i = 0; i < IC * KH; i++) \ - dma_queue_pop(dma_q); \ + dma_queue_flush(dma_q); \ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, r); \ for (uint32_t iow = 0; iow < OW; iow++) { \ DST_CTYPE * dst_patch = dstb + (uint64_t) iow * patch_stride; \ diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 7b7e6cae10da..9a8d46bfdb4d 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -9789,6 +9789,14 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, {2, 2, 1536, 729}, {2, 2, 1536, 4096}, 1, 1, 0, 0, 1, 1, true)); test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {128, 128, 1, 2}, {32, 33, 1, 2}, 1, 1, 1, 1, 1, 1, true)); test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {128, 128, 2, 1}, {33, 34, 2, 1}, 1, 1, 1, 1, 1, 1, true)); + // non-overlapping (stride == kernel, no pad/dilation) with IC*KH well above 256, e.g. SD1.5 1x1 convs + for (ggml_type dst_type : {GGML_TYPE_F16, GGML_TYPE_F32}) { + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, dst_type, {64, 64, 320, 1}, {1, 1, 320, 1}, 1, 1, 0, 0, 1, 1, true)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, dst_type, {16, 16, 1280, 2}, {1, 1, 1280, 1}, 1, 1, 0, 0, 1, 1, true)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, dst_type, {8, 8, 2560, 1}, {1, 1, 2560, 1}, 1, 1, 0, 0, 1, 1, true)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, dst_type, {32, 32, 192, 1}, {2, 2, 192, 1}, 2, 2, 0, 0, 1, 1, true)); + test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, dst_type, {64, 512, 1, 1}, {2, 512, 1, 1}, 2, 0, 0, 0, 1, 0, false)); + } // im2col 3D test_cases.emplace_back(new test_im2col_3d(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32)); From 8a1a9b5126126e5228b95fa909d4b08fac65e8b3 Mon Sep 17 00:00:00 2001 From: ynankani Date: Fri, 9 Oct 2026 06:32:52 +0000 Subject: [PATCH 003/431] CUDA: pass src1 precision to host MMQ config helpers (#30168) * CUDA: pass src1 precision to host MMQ config helpers Signed-off-by: ynankani * make src1 prec explicit Signed-off-by: ynankani --------- Signed-off-by: ynankani --- ggml/src/ggml-cuda/mmq-load-tiles.cuh | 128 ++++++++--------- ggml/src/ggml-cuda/mmq-vec-dot.cuh | 80 +++++------ ggml/src/ggml-cuda/mmq.cu | 44 +++--- ggml/src/ggml-cuda/mmq.cuh | 196 +++++++++++++------------- 4 files changed, 223 insertions(+), 225 deletions(-) diff --git a/ggml/src/ggml-cuda/mmq-load-tiles.cuh b/ggml/src/ggml-cuda/mmq-load-tiles.cuh index d5654868bc08..0e59c0347b72 100644 --- a/ggml/src/ggml-cuda/mmq-load-tiles.cuh +++ b/ggml/src/ggml-cuda/mmq-load-tiles.cuh @@ -7,9 +7,9 @@ template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q1_0( const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) int * x_qs = (int *) x_tile; @@ -98,9 +98,9 @@ template static __device__ __forceinline_ template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q2_0( const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) int * x_qs = (int *) x_tile; @@ -187,9 +187,9 @@ template static __device__ __forceinline_ template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_0( const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) int * x_qs = (int *) x_tile; @@ -250,9 +250,9 @@ template static __device__ __forceinline_ template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_1( const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) int * x_qs = (int *) x_tile; @@ -313,9 +313,9 @@ template static __device__ __forceinline_ template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q5_0( const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) int * x_qs = (int *) x_tile; @@ -393,9 +393,9 @@ template static __device__ __forceinline_ template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q5_1( const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) int * x_qs = (int *) x_tile; @@ -471,9 +471,9 @@ template static __device__ __forceinline_ template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q8_0( const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) int * x_qs = (int *) x_tile; @@ -537,9 +537,9 @@ template static __device__ __forceinline_ template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q2_K( const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) int * x_qs = (int *) x_tile; @@ -598,9 +598,9 @@ template static __device__ __forceinline_ template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q3_K( const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) int * x_qs = (int *) x_tile; @@ -711,9 +711,9 @@ static __device__ __forceinline__ int unpack_scales_q45_K(const int * scales, co template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_K( const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) int * x_qs = (int *) x_tile; @@ -822,9 +822,9 @@ template static __device__ __forceinline_ template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q5_K( const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) int * x_qs = (int *) x_tile; @@ -946,9 +946,9 @@ template static __device__ __forceinline_ template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q6_K( const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) int * x_qs = (int *) x_tile; @@ -1036,9 +1036,9 @@ template static __device__ __forceinline_ template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq1_s( const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) int * x_qs = (int *) x_tile; @@ -1098,9 +1098,9 @@ template static __device__ __forceinline_ template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_xxs( const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) int * x_qs = (int *) x_tile; @@ -1162,9 +1162,9 @@ template static __device__ __forceinline_ template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_xs( const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) int * x_qs = (int *) x_tile; @@ -1227,9 +1227,9 @@ template static __device__ __forceinline_ template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_s( const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) int * x_qs = (int *) x_tile; @@ -1295,9 +1295,9 @@ template static __device__ __forceinline_ template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq3_xxs( const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) int * x_qs = (int *) x_tile; @@ -1359,9 +1359,9 @@ template static __device__ __forceinline_ template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq3_s( const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) int * x_qs = (int *) x_tile; @@ -1428,9 +1428,9 @@ template static __device__ __forceinline_ template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq4_xs( const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) int * x_qs = (int *) x_tile; @@ -1495,9 +1495,9 @@ template static __device__ __forceinline_ template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq4_nl( const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) int * x_qs = (int *) x_tile; @@ -1564,9 +1564,9 @@ template static __device__ __forceinline_ template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_mxfp4( const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) int * x_qs = (int *) x_tile; @@ -1670,7 +1670,7 @@ template static __device__ __forceinline_ } } -template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_nvfp4( +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_nvfp4( const char * __restrict__ x, int * __restrict__ x_tile, const int kb0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size; diff --git a/ggml/src/ggml-cuda/mmq-vec-dot.cuh b/ggml/src/ggml-cuda/mmq-vec-dot.cuh index b39e4d579a8b..a75dd046e5c3 100644 --- a/ggml/src/ggml-cuda/mmq-vec-dot.cuh +++ b/ggml/src/ggml-cuda/mmq-vec-dot.cuh @@ -10,8 +10,8 @@ using namespace ggml_cuda_mma; template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q4_0_q8_1_dp4a( const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_0, I); const int * x_qs = (const int *) x; @@ -60,8 +60,8 @@ template static __device__ __forceinline_ template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q4_1_q8_1_dp4a( const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_1, I); const int * x_qs = (const int *) x; @@ -110,8 +110,8 @@ template static __device__ __forceinline_ template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a( const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, I); const int * x_qs = (const int *) x; @@ -148,8 +148,8 @@ static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma( typedef tile<16, 8, int, input_layout> tile_B; typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); - constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); @@ -203,8 +203,8 @@ static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma( typedef tile< 8, 8, int> tile_B; typedef tile<16, 8, int> tile_C; - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); - constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); @@ -281,8 +281,8 @@ static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma( template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a( const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_1, I); const int * x_qs = (const int *) x; @@ -318,8 +318,8 @@ template static __device__ __forceinline_ typedef tile<16, 8, int, input_layout> tile_B; typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); - constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); @@ -368,8 +368,8 @@ template static __device__ __forceinline_ typedef tile< 8, 8, int> tile_B; typedef tile<16, 8, int> tile_C; - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); - constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); @@ -442,8 +442,8 @@ template static __device__ __forceinline_ template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a( const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(type, I); const int * x_qs = (const int *) x; @@ -474,7 +474,7 @@ template static __device__ __forceinline_ } // Used for Q3_K, IQ2_S, and IQ2_XS: -template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma( +template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma( const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { #if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) constexpr data_layout input_layout = get_input_data_layout(); @@ -483,7 +483,7 @@ template tile_C; constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, prec_src1); - constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, prec_src1); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); @@ -533,7 +533,7 @@ template tile_C; constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, prec_src1); - constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, prec_src1); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); @@ -610,8 +610,8 @@ template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q2_K_q8_1_dp4a( const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q2_K, I); const int * x_qs = (const int *) x; @@ -680,8 +680,8 @@ template static __device__ __forceinline_ typedef tile<16, 4, int, input_layout> tile_B; typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); - constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); @@ -749,8 +749,8 @@ template static __device__ __forceinline_ typedef tile< 8, 4, int> tile_B; typedef tile<16, 8, int> tile_C; - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); - constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); @@ -870,8 +870,8 @@ template static __device__ __forceinline_ template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q3_K_q8_1_dp4a( const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q3_K, I); const int * x_qs = (const int *) x; @@ -905,8 +905,8 @@ template static __device__ __forceinline_ template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q4_K_q8_1_dp4a( const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_K, I); const int * x_qs = (const int *) x; @@ -940,8 +940,8 @@ template static __device__ __forceinline_ template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q5_K_q8_1_dp4a( const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_K, I); const int * x_qs = (const int *) x; @@ -975,8 +975,8 @@ template static __device__ __forceinline_ template static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q6_K_q8_1_dp4a( const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q6_K, I); const int * x_qs = (const int *) x; @@ -1015,8 +1015,8 @@ template static __device__ __forceinline_ typedef tile<16, 4, int, input_layout> tile_B; typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C; - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); - constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); @@ -1066,8 +1066,8 @@ template static __device__ __forceinline_ typedef tile< 8, 4, int> tile_B; typedef tile<16, 8, int> tile_C; - constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); - constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); @@ -1181,7 +1181,7 @@ template static __device__ __forceinline_ typedef tile<16, 8, float> tile_C; constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q4); - constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q4); constexpr int ntx = rows_per_warp / tile_C::I; constexpr int nfrags = MMQ_TILE_NE_K / tile_A::J; diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu index 027a7e60da23..91b89999ef8e 100644 --- a/ggml/src/ggml-cuda/mmq.cu +++ b/ggml/src/ggml-cuda/mmq.cu @@ -8,66 +8,66 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream, const ggml_prec prec_src1) { switch (args.type_x) { case GGML_TYPE_Q1_0: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_Q2_0: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_Q4_0: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_Q4_1: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_Q5_0: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_Q5_1: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_Q8_0: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; // ----------------------------------------------------------------------- case GGML_TYPE_Q2_K: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_Q3_K: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_Q4_K: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_Q5_K: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_Q6_K: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; // ----------------------------------------------------------------------- case GGML_TYPE_IQ1_S: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_IQ2_XXS: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_IQ2_XS: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_IQ2_S: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_IQ3_XXS: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_IQ3_S: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_IQ4_XS: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_IQ4_NL: - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; // ----------------------------------------------------------------------- case GGML_TYPE_MXFP4: @@ -76,14 +76,14 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, con mul_mat_q_case(ctx, args, stream); break; } - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_NVFP4: if (prec_src1 == GGML_PREC_Q4) { mul_mat_q_case(ctx, args, stream); break; } - mul_mat_q_case(ctx, args, stream); + mul_mat_q_case(ctx, args, stream); break; default: GGML_ABORT("fatal error"); diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh index 198aab4b5044..d3e7b75ea7c8 100644 --- a/ggml/src/ggml-cuda/mmq.cuh +++ b/ggml/src/ggml-cuda/mmq.cuh @@ -227,7 +227,7 @@ struct ggml_cuda_mmq_config { #undef CASE -static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1 = GGML_PREC_Q8) { +static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { if (GGML_CUDA_CC_IS_AMD(cc)) { if (GGML_CUDA_CC_IS_GCN(cc)) { return ggml_cuda_mmq_get_config_gcn(type, J, fallback); @@ -262,7 +262,7 @@ static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type ty return ggml_cuda_mmq_get_config_pascal_older(type, J, fallback); } -static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8) { +static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { #ifdef GGML_USE_HIP #ifdef GCN return ggml_cuda_mmq_get_config_gcn(type, J, fallback); @@ -295,79 +295,77 @@ static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_t GGML_UNUSED_VARS(type, J, fallback, prec_src1); } -// FIXME all of the host functions are missing prec_src1, this can lead to inconsitent behavior. - -static __host__ int ggml_cuda_mmq_get_type(const ggml_type type, const int J, const bool fallback, const int cc) { - return ggml_cuda_mmq_get_config(type, J, fallback, cc).type; +static __host__ int ggml_cuda_mmq_get_type(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).type; } -static constexpr __device__ int ggml_cuda_mmq_get_type(ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8) { +static constexpr __device__ int ggml_cuda_mmq_get_type(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).type; } -static constexpr __device__ int ggml_cuda_mmq_get_nthreads(ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8) { +static constexpr __device__ int ggml_cuda_mmq_get_nthreads(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).nthreads; } -static constexpr __device__ int ggml_cuda_mmq_get_occupancy(ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8) { +static constexpr __device__ int ggml_cuda_mmq_get_occupancy(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).occupancy; } -static __host__ int ggml_cuda_mmq_get_I(const ggml_type type, const int J, const bool fallback, const int cc) { - return ggml_cuda_mmq_get_config(type, J, fallback, cc).I; +static __host__ int ggml_cuda_mmq_get_I(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).I; } -static constexpr __device__ int ggml_cuda_mmq_get_I(ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8) { +static constexpr __device__ int ggml_cuda_mmq_get_I(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).I; } -static __host__ int ggml_cuda_mmq_get_J(const ggml_type type, const int J, const bool fallback, const int cc) { - return ggml_cuda_mmq_get_config(type, J, fallback, cc).J; +static __host__ int ggml_cuda_mmq_get_J(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).J; } -static constexpr __device__ int ggml_cuda_mmq_get_J(ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8) { +static constexpr __device__ int ggml_cuda_mmq_get_J(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).J; } -static __host__ ggml_cuda_mmq_sram_layout ggml_cuda_mmq_get_sram_layout(const ggml_type type, const int J, const bool fallback, const int cc) { - return ggml_cuda_mmq_get_config(type, J, fallback, cc).sram_layout; +static __host__ ggml_cuda_mmq_sram_layout ggml_cuda_mmq_get_sram_layout(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).sram_layout; } -static constexpr __device__ ggml_cuda_mmq_sram_layout ggml_cuda_mmq_get_sram_layout(ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8) { +static constexpr __device__ ggml_cuda_mmq_sram_layout ggml_cuda_mmq_get_sram_layout(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).sram_layout; } -static __host__ int ggml_cuda_mmq_get_K_vram(const ggml_type type, const int J, const bool fallback, const int cc) { - return ggml_cuda_mmq_get_config(type, J, fallback, cc).K_vram; +static __host__ int ggml_cuda_mmq_get_K_vram(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).K_vram; } -static constexpr __device__ int ggml_cuda_mmq_get_K_vram(ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8) { +static constexpr __device__ int ggml_cuda_mmq_get_K_vram(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).K_vram; } -static __host__ bool ggml_cuda_mmq_get_stream_k(const ggml_type type, const int J, const bool fallback, const int cc) { - return ggml_cuda_mmq_get_config(type, J, fallback, cc).stream_k; +static __host__ bool ggml_cuda_mmq_get_stream_k(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).stream_k; } -static constexpr __device__ bool ggml_cuda_mmq_get_stream_k(ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8) { +static constexpr __device__ bool ggml_cuda_mmq_get_stream_k(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).stream_k; } -static __host__ int ggml_cuda_mmq_get_fallback(const ggml_type type, const int J, const bool fallback, const int cc) { - return ggml_cuda_mmq_get_config(type, J, fallback, cc).fallback; +static __host__ int ggml_cuda_mmq_get_fallback(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).fallback; } -static constexpr __device__ int ggml_cuda_mmq_get_fallback(ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8) { +static constexpr __device__ int ggml_cuda_mmq_get_fallback(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).fallback; } // --------------------------------------------------------------------------------------------- -static __host__ int ggml_cuda_mmq_get_sram_stride(const ggml_type type, const int J, const bool fallback, const int cc) { - return ggml_cuda_mmq_get_sram_stride(ggml_cuda_mmq_get_sram_layout(type, J, fallback, cc)); +static __host__ int ggml_cuda_mmq_get_sram_stride(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { + return ggml_cuda_mmq_get_sram_stride(ggml_cuda_mmq_get_sram_layout(type, J, fallback, cc, prec_src1)); } -static constexpr __device__ int ggml_cuda_mmq_get_sram_stride(ggml_type type, int J, bool fallback, ggml_prec prec_src1 = GGML_PREC_Q8) { +static constexpr __device__ int ggml_cuda_mmq_get_sram_stride(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { return ggml_cuda_mmq_get_sram_stride(ggml_cuda_mmq_get_sram_layout(type, J, fallback, prec_src1)); } @@ -375,8 +373,8 @@ static __host__ bool ggml_cuda_mmq_needs_fallback(const int64_t nrows_x) { return nrows_x % 128 != 0; } -static constexpr __device__ int ggml_cuda_mmq_get_rows_per_warp(ggml_type type, int J, bool fallback) { - return ggml_cuda_mmq_get_config(type, J, fallback).rows_per_warp(); +static constexpr __device__ int ggml_cuda_mmq_get_rows_per_warp(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { + return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).rows_per_warp(); } #define MMQ_DP4A_TXS_Q4_0 tile_x_sizes{I*MMQ_TILE_NE_K + I, I*MMQ_TILE_NE_K/QI4_0 + I/QI4_0, 0} @@ -432,12 +430,12 @@ static __host__ int ggml_cuda_mmq_get_nbytes_shared_x(const ggml_cuda_mmq_config #include "mmq-load-tiles.cuh" #include "mmq-vec-dot.cuh" -template static __device__ __forceinline__ void ggml_cuda_mmq_write_back_dp4a( +template static __device__ __forceinline__ void ggml_cuda_mmq_write_back_dp4a( const float * __restrict__ sum, const int32_t * __restrict__ ids_dst, float * __restrict__ dst, const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; - constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1); const bool y_scale_used = y_scale != nullptr; @@ -471,7 +469,7 @@ template static __device__ __forceinline_ } } -template +template static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma( const float * __restrict__ sum, const int * __restrict__ ids_dst, float * __restrict__ dst, const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max) { @@ -482,7 +480,7 @@ static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma( typedef tile<16, 8, int> tile_C; #endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) - constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); + constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, prec_src1); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. const int i0 = (threadIdx.y / ntx) * (ntx*tile_C::I); @@ -536,7 +534,7 @@ struct ggml_cuda_mmq_util_funcs { vdr(vdr), load_tiles(load_tiles), vec_dot(vec_dot), write_back(write_back) {} }; -template +template static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_funcs() { if (!ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).use_mma_data_layout()) { switch (type) { @@ -545,136 +543,136 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_func VDR_Q1_0_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_q1_0, ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, - ggml_cuda_mmq_write_back_dp4a); + ggml_cuda_mmq_write_back_dp4a); case GGML_TYPE_Q2_0: return ggml_cuda_mmq_util_funcs( VDR_Q2_0_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_q2_0, ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, - ggml_cuda_mmq_write_back_dp4a); + ggml_cuda_mmq_write_back_dp4a); case GGML_TYPE_Q4_0: return ggml_cuda_mmq_util_funcs( VDR_Q4_0_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_q4_0, ggml_cuda_mmq_vec_dot_q4_0_q8_1_dp4a, - ggml_cuda_mmq_write_back_dp4a); + ggml_cuda_mmq_write_back_dp4a); case GGML_TYPE_Q4_1: return ggml_cuda_mmq_util_funcs( VDR_Q4_1_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_q4_1, ggml_cuda_mmq_vec_dot_q4_1_q8_1_dp4a, - ggml_cuda_mmq_write_back_dp4a); + ggml_cuda_mmq_write_back_dp4a); case GGML_TYPE_Q5_0: return ggml_cuda_mmq_util_funcs( VDR_Q5_0_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_q5_0, ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, - ggml_cuda_mmq_write_back_dp4a); + ggml_cuda_mmq_write_back_dp4a); case GGML_TYPE_Q5_1: return ggml_cuda_mmq_util_funcs( VDR_Q5_1_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_q5_1, ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a, - ggml_cuda_mmq_write_back_dp4a); + ggml_cuda_mmq_write_back_dp4a); case GGML_TYPE_Q8_0: return ggml_cuda_mmq_util_funcs( VDR_Q8_0_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_q8_0, ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, - ggml_cuda_mmq_write_back_dp4a); + ggml_cuda_mmq_write_back_dp4a); // --------------------------------------------------------------------------------------------- case GGML_TYPE_Q2_K: return ggml_cuda_mmq_util_funcs( VDR_Q2_K_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_q2_K, ggml_cuda_mmq_vec_dot_q2_K_q8_1_dp4a, - ggml_cuda_mmq_write_back_dp4a); + ggml_cuda_mmq_write_back_dp4a); case GGML_TYPE_Q3_K: return ggml_cuda_mmq_util_funcs( VDR_Q3_K_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_q3_K, ggml_cuda_mmq_vec_dot_q3_K_q8_1_dp4a, - ggml_cuda_mmq_write_back_dp4a); + ggml_cuda_mmq_write_back_dp4a); case GGML_TYPE_Q4_K: return ggml_cuda_mmq_util_funcs( VDR_Q4_K_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_q4_K, ggml_cuda_mmq_vec_dot_q4_K_q8_1_dp4a, - ggml_cuda_mmq_write_back_dp4a); + ggml_cuda_mmq_write_back_dp4a); case GGML_TYPE_Q5_K: return ggml_cuda_mmq_util_funcs( VDR_Q5_K_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_q5_K, ggml_cuda_mmq_vec_dot_q5_K_q8_1_dp4a, - ggml_cuda_mmq_write_back_dp4a); + ggml_cuda_mmq_write_back_dp4a); case GGML_TYPE_Q6_K: return ggml_cuda_mmq_util_funcs( VDR_Q6_K_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_q6_K, ggml_cuda_mmq_vec_dot_q6_K_q8_1_dp4a, - ggml_cuda_mmq_write_back_dp4a); + ggml_cuda_mmq_write_back_dp4a); // --------------------------------------------------------------------------------------------- case GGML_TYPE_IQ1_S: return ggml_cuda_mmq_util_funcs( VDR_IQ1_S_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_iq1_s, ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a, - ggml_cuda_mmq_write_back_dp4a); + ggml_cuda_mmq_write_back_dp4a); case GGML_TYPE_IQ2_XXS: return ggml_cuda_mmq_util_funcs( VDR_IQ2_XXS_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_iq2_xxs, ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, - ggml_cuda_mmq_write_back_dp4a); + ggml_cuda_mmq_write_back_dp4a); case GGML_TYPE_IQ2_XS: return ggml_cuda_mmq_util_funcs( VDR_IQ2_XS_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_iq2_xs, ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a, - ggml_cuda_mmq_write_back_dp4a); + ggml_cuda_mmq_write_back_dp4a); case GGML_TYPE_IQ2_S: return ggml_cuda_mmq_util_funcs( VDR_IQ2_S_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_iq2_s, ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a, - ggml_cuda_mmq_write_back_dp4a); + ggml_cuda_mmq_write_back_dp4a); case GGML_TYPE_IQ3_XXS: return ggml_cuda_mmq_util_funcs( VDR_IQ3_XXS_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_iq3_xxs, ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, - ggml_cuda_mmq_write_back_dp4a); + ggml_cuda_mmq_write_back_dp4a); case GGML_TYPE_IQ3_S: return ggml_cuda_mmq_util_funcs( VDR_IQ3_S_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_iq3_s, ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, - ggml_cuda_mmq_write_back_dp4a); + ggml_cuda_mmq_write_back_dp4a); case GGML_TYPE_IQ4_XS: return ggml_cuda_mmq_util_funcs( VDR_IQ4_XS_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_iq4_xs, ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, - ggml_cuda_mmq_write_back_dp4a); + ggml_cuda_mmq_write_back_dp4a); case GGML_TYPE_IQ4_NL: return ggml_cuda_mmq_util_funcs( VDR_IQ4_NL_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_iq4_nl, ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, - ggml_cuda_mmq_write_back_dp4a); + ggml_cuda_mmq_write_back_dp4a); // --------------------------------------------------------------------------------------------- case GGML_TYPE_MXFP4: return ggml_cuda_mmq_util_funcs( VDR_MXFP4_Q8_1_MMQ, ggml_cuda_mmq_load_tiles_mxfp4, ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, - ggml_cuda_mmq_write_back_dp4a); + ggml_cuda_mmq_write_back_dp4a); case GGML_TYPE_NVFP4: return ggml_cuda_mmq_util_funcs( VDR_NVFP4_Q8_1_MMQ, - ggml_cuda_mmq_load_tiles_nvfp4, + ggml_cuda_mmq_load_tiles_nvfp4, ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a, - ggml_cuda_mmq_write_back_dp4a); + ggml_cuda_mmq_write_back_dp4a); default: return ggml_cuda_mmq_util_funcs(1, nullptr, nullptr, nullptr); } @@ -690,7 +688,7 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_func -1, ggml_cuda_mmq_load_tiles_mxfp4_fp4, ggml_cuda_mmq_vec_dot_fp4_fp4_mma, - ggml_cuda_mmq_write_back_mma); + ggml_cuda_mmq_write_back_mma); } break; case GGML_TYPE_NVFP4: @@ -699,7 +697,7 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_func -1, ggml_cuda_mmq_load_tiles_nvfp4_nvfp4, ggml_cuda_mmq_vec_dot_fp4_fp4_mma, - ggml_cuda_mmq_write_back_mma); + ggml_cuda_mmq_write_back_mma); } break; default: @@ -715,164 +713,164 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_func -1, ggml_cuda_mmq_load_tiles_q1_0, ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, - ggml_cuda_mmq_write_back_mma); + ggml_cuda_mmq_write_back_mma); case GGML_TYPE_Q2_0: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_q2_0, ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, - ggml_cuda_mmq_write_back_mma); + ggml_cuda_mmq_write_back_mma); case GGML_TYPE_Q4_0: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_q4_0, ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, - ggml_cuda_mmq_write_back_mma); + ggml_cuda_mmq_write_back_mma); case GGML_TYPE_Q4_1: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_q4_1, ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma, - ggml_cuda_mmq_write_back_mma); + ggml_cuda_mmq_write_back_mma); case GGML_TYPE_Q5_0: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_q5_0, ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, - ggml_cuda_mmq_write_back_mma); + ggml_cuda_mmq_write_back_mma); case GGML_TYPE_Q5_1: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_q5_1, ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma, - ggml_cuda_mmq_write_back_mma); + ggml_cuda_mmq_write_back_mma); case GGML_TYPE_Q8_0: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_q8_0, ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, - ggml_cuda_mmq_write_back_mma); + ggml_cuda_mmq_write_back_mma); // --------------------------------------------------------------------------------------------- case GGML_TYPE_Q2_K: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_q2_K, ggml_cuda_mmq_vec_dot_q2_K_q8_1_mma, - ggml_cuda_mmq_write_back_mma); + ggml_cuda_mmq_write_back_mma); case GGML_TYPE_Q3_K: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_q3_K, - ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma, - ggml_cuda_mmq_write_back_mma); + ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma, + ggml_cuda_mmq_write_back_mma); case GGML_TYPE_Q4_K: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_q4_K, ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma, - ggml_cuda_mmq_write_back_mma); + ggml_cuda_mmq_write_back_mma); case GGML_TYPE_Q5_K: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_q5_K, ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma, - ggml_cuda_mmq_write_back_mma); + ggml_cuda_mmq_write_back_mma); case GGML_TYPE_Q6_K: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_q6_K, ggml_cuda_mmq_vec_dot_q6_K_q8_1_mma, - ggml_cuda_mmq_write_back_mma); + ggml_cuda_mmq_write_back_mma); // --------------------------------------------------------------------------------------------- case GGML_TYPE_IQ1_S: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_iq1_s, ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma, - ggml_cuda_mmq_write_back_mma); + ggml_cuda_mmq_write_back_mma); case GGML_TYPE_IQ2_XXS: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_iq2_xxs, ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, - ggml_cuda_mmq_write_back_mma); + ggml_cuda_mmq_write_back_mma); case GGML_TYPE_IQ2_XS: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_iq2_xs, - ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma, - ggml_cuda_mmq_write_back_mma); + ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma, + ggml_cuda_mmq_write_back_mma); case GGML_TYPE_IQ2_S: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_iq2_s, - ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma, - ggml_cuda_mmq_write_back_mma); + ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma, + ggml_cuda_mmq_write_back_mma); case GGML_TYPE_IQ3_XXS: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_iq3_xxs, ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, - ggml_cuda_mmq_write_back_mma); + ggml_cuda_mmq_write_back_mma); case GGML_TYPE_IQ3_S: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_iq3_s, ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, - ggml_cuda_mmq_write_back_mma); + ggml_cuda_mmq_write_back_mma); case GGML_TYPE_IQ4_XS: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_iq4_xs, ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, - ggml_cuda_mmq_write_back_mma); + ggml_cuda_mmq_write_back_mma); case GGML_TYPE_IQ4_NL: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_iq4_nl, ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, - ggml_cuda_mmq_write_back_mma); + ggml_cuda_mmq_write_back_mma); // --------------------------------------------------------------------------------------------- case GGML_TYPE_MXFP4: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_mxfp4, ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, - ggml_cuda_mmq_write_back_mma); + ggml_cuda_mmq_write_back_mma); case GGML_TYPE_NVFP4: return ggml_cuda_mmq_util_funcs( -1, ggml_cuda_mmq_load_tiles_nvfp4, ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma, - ggml_cuda_mmq_write_back_mma); + ggml_cuda_mmq_write_back_mma); default: return ggml_cuda_mmq_util_funcs(1, nullptr, nullptr, nullptr); } } -template +template static constexpr __device__ int ggml_cuda_mmq_get_vdr() { return ggml_cuda_mmq_get_util_funcs().vdr; } -template +template static constexpr __device__ ggml_cuda_mmq_load_tiles_t ggml_cuda_mmq_get_load_tiles() { return ggml_cuda_mmq_get_util_funcs().load_tiles; } -template +template static constexpr __device__ ggml_cuda_mmq_vec_dot_t ggml_cuda_mmq_get_vec_dot() { return ggml_cuda_mmq_get_util_funcs().vec_dot; } -template +template static constexpr __device__ ggml_cuda_mmq_write_back_t ggml_cuda_mmq_get_write_back() { return ggml_cuda_mmq_get_util_funcs().write_back; } // --------------------------------------------------------------------------------------------- -template +template static __device__ __forceinline__ void mul_mat_q_process_tile( const char * __restrict__ x, const int offset_x, const int * __restrict__ y, const int * __restrict__ ids_dst, float * __restrict__ dst, float * __restrict__ tmp_fixup, @@ -953,7 +951,7 @@ static __device__ __forceinline__ void mul_mat_q_process_tile( // The mul_mat_q kernel implements "stream-k" work partitioning as described in https://arxiv.org/abs/2301.03598 -template +template __launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1), ggml_cuda_mmq_get_occupancy(type, J, fallback, prec_src1)) static __global__ void mul_mat_q( const char * __restrict__ x, const int * __restrict__ y, const int32_t * __restrict__ ids_dst, @@ -1240,7 +1238,7 @@ static __global__ void mul_mat_q( tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop); } -template +template __launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1)/2, 1) static __global__ void mul_mat_q_stream_k_fixup( const int32_t * __restrict__ ids_dst, const int32_t * __restrict__ expert_bounds, float * __restrict__ dst, @@ -1395,7 +1393,7 @@ static size_t mmq_get_nbytes_shared(const ggml_cuda_mmq_config & config, const i return nbs_ids + nbs_x + GGML_PAD(nbs_y, config.nthreads*sizeof(int)); } -template +template static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { const int id = ggml_cuda_get_device(); const int cc = ggml_cuda_info().devices[id].cc; @@ -1477,7 +1475,7 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a ntx_fd); } -template +template void mul_mat_q_switch_J(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { switch (args.J_best) { case 8: @@ -1535,7 +1533,7 @@ void mul_mat_q_switch_J(ggml_backend_cuda_context & ctx, const mmq_args & args, } } -template +template void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { if (ggml_cuda_mmq_needs_fallback(args.nrows_x)) { constexpr bool fallback = true; @@ -1547,7 +1545,7 @@ void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cuda } #define DECL_MMQ_CASE(type) \ - template void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) \ + template void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) \ // FP4 variant: uses native FP4 MMA instead of keeping src1 at Q8_1. #define DECL_MMQ_CASE_W4A4(type) \ From 1f8fa52318b594b373e7447a3c2bb36cdba9515f Mon Sep 17 00:00:00 2001 From: Pascal Date: Fri, 9 Oct 2026 10:15:32 +0200 Subject: [PATCH 004/431] metal: add the 128/96 flash attention kernels, accept every compiled pair (#30209) --- ggml/src/ggml-metal/ggml-metal-device.m | 11 +++++++---- ggml/src/ggml-metal/ggml-metal-ops.cpp | 1 + ggml/src/ggml-metal/ggml-metal-tuning.cpp | 3 +++ ggml/src/ggml-metal/kernels/fa_f16.metal | 2 ++ ggml/src/ggml-metal/kernels/fa_f32.metal | 1 + ggml/src/ggml-metal/kernels/fa_q4_0.metal | 1 + ggml/src/ggml-metal/kernels/fa_q4_1.metal | 1 + ggml/src/ggml-metal/kernels/fa_q5_0.metal | 1 + ggml/src/ggml-metal/kernels/fa_q5_1.metal | 1 + ggml/src/ggml-metal/kernels/fa_q8_0.metal | 1 + ggml/src/ggml-metal/kernels/fa_vec_f16.metal | 4 ++++ ggml/src/ggml-metal/kernels/fa_vec_f32.metal | 1 + ggml/src/ggml-metal/kernels/fa_vec_q4_0.metal | 3 +++ ggml/src/ggml-metal/kernels/fa_vec_q4_1.metal | 3 +++ ggml/src/ggml-metal/kernels/fa_vec_q5_0.metal | 3 +++ ggml/src/ggml-metal/kernels/fa_vec_q5_1.metal | 3 +++ ggml/src/ggml-metal/kernels/fa_vec_q8_0.metal | 3 +++ 17 files changed, 39 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index 827d92622d7c..13aecbdd320a 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -1741,10 +1741,13 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te op->src[0]->ne[0] != 576) { return false; } - if (op->src[1]->ne[0] == 72 && op->src[1]->ne[0] != op->src[2]->ne[0]) { - return false; - } - if (op->src[1]->ne[0] < op->src[2]->ne[0]) { + // the kernels exist for K == V and for these K > V pairs only + if (op->src[1]->ne[0] != op->src[2]->ne[0] && + !(op->src[1]->ne[0] == 96 && op->src[2]->ne[0] == 64) && + !(op->src[1]->ne[0] == 128 && op->src[2]->ne[0] == 96) && + !(op->src[1]->ne[0] == 192 && op->src[2]->ne[0] == 128) && + !(op->src[1]->ne[0] == 320 && op->src[2]->ne[0] == 256) && + !(op->src[1]->ne[0] == 576 && op->src[2]->ne[0] == 512)) { return false; } if (op->src[1]->type != op->src[2]->type) { diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index 597d5300c071..86b9fe4ef360 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -3180,6 +3180,7 @@ static int ggml_metal_op_flash_attn_ext_n_kv_max_sparse(const ggml_tensor * op) (dk == 96 && dv == 96) || (dk == 96 && dv == 64) || (dk == 128 && dv == 128) || + (dk == 128 && dv == 96) || (dk == 192 && dv == 128) || (dk == 192 && dv == 192) || (dk == 256 && dv == 256) || diff --git a/ggml/src/ggml-metal/ggml-metal-tuning.cpp b/ggml/src/ggml-metal/ggml-metal-tuning.cpp index 217ad43b311c..0fe50277ea78 100644 --- a/ggml/src/ggml-metal/ggml-metal-tuning.cpp +++ b/ggml/src/ggml-metal/ggml-metal-tuning.cpp @@ -40,6 +40,9 @@ int fa_vec_baseline_ne(int dk, int dv) { if (dk == 128 && dv == 128) { return 1; } + if (dk == 128 && dv == 96) { + return 4; + } if (dk == 192 && dv == 192) { return 2; } diff --git a/ggml/src/ggml-metal/kernels/fa_f16.metal b/ggml/src/ggml-metal/kernels/fa_f16.metal index 2dfd3799d137..0f44b419bdea 100644 --- a/ggml/src/ggml-metal/kernels/fa_f16.metal +++ b/ggml/src/ggml-metal/kernels/fa_f16.metal @@ -44,6 +44,7 @@ template [[host_name("kernel_flash_attn_ext_f16_dk96_dv96" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_f16_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_f16_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_f16_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f16_dk128_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_f16_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_f16_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_f16_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; @@ -62,6 +63,7 @@ template [[host_name("kernel_flash_attn_ext_bf16_dk96_dv96" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_bf16_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_bf16_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_bf16_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_bf16_dk128_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_bf16_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_bf16_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_bf16_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; diff --git a/ggml/src/ggml-metal/kernels/fa_f32.metal b/ggml/src/ggml-metal/kernels/fa_f32.metal index 8d38d775229e..449079e8f6e5 100644 --- a/ggml/src/ggml-metal/kernels/fa_f32.metal +++ b/ggml/src/ggml-metal/kernels/fa_f32.metal @@ -41,6 +41,7 @@ template [[host_name("kernel_flash_attn_ext_f32_dk96_dv96" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_f32_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_f32_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_f32_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_f32_dk128_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_f32_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_f32_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_f32_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; diff --git a/ggml/src/ggml-metal/kernels/fa_q4_0.metal b/ggml/src/ggml-metal/kernels/fa_q4_0.metal index 49bb4c13cf7b..6b05b5df6bdf 100644 --- a/ggml/src/ggml-metal/kernels/fa_q4_0.metal +++ b/ggml/src/ggml-metal/kernels/fa_q4_0.metal @@ -41,6 +41,7 @@ template [[host_name("kernel_flash_attn_ext_q4_0_dk96_dv96" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_q4_0_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q4_0_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q4_0_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_0_dk128_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q4_0_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q4_0_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q4_0_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; diff --git a/ggml/src/ggml-metal/kernels/fa_q4_1.metal b/ggml/src/ggml-metal/kernels/fa_q4_1.metal index 79797ffced9d..87fac701ea92 100644 --- a/ggml/src/ggml-metal/kernels/fa_q4_1.metal +++ b/ggml/src/ggml-metal/kernels/fa_q4_1.metal @@ -41,6 +41,7 @@ template [[host_name("kernel_flash_attn_ext_q4_1_dk96_dv96" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_q4_1_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q4_1_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q4_1_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q4_1_dk128_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q4_1_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q4_1_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q4_1_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; diff --git a/ggml/src/ggml-metal/kernels/fa_q5_0.metal b/ggml/src/ggml-metal/kernels/fa_q5_0.metal index 93de514b425b..cd0e31f715a7 100644 --- a/ggml/src/ggml-metal/kernels/fa_q5_0.metal +++ b/ggml/src/ggml-metal/kernels/fa_q5_0.metal @@ -41,6 +41,7 @@ template [[host_name("kernel_flash_attn_ext_q5_0_dk96_dv96" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_q5_0_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q5_0_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q5_0_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_0_dk128_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q5_0_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q5_0_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q5_0_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; diff --git a/ggml/src/ggml-metal/kernels/fa_q5_1.metal b/ggml/src/ggml-metal/kernels/fa_q5_1.metal index 3918c571e20c..4d74951acc5f 100644 --- a/ggml/src/ggml-metal/kernels/fa_q5_1.metal +++ b/ggml/src/ggml-metal/kernels/fa_q5_1.metal @@ -41,6 +41,7 @@ template [[host_name("kernel_flash_attn_ext_q5_1_dk96_dv96" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_q5_1_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q5_1_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q5_1_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q5_1_dk128_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q5_1_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q5_1_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q5_1_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; diff --git a/ggml/src/ggml-metal/kernels/fa_q8_0.metal b/ggml/src/ggml-metal/kernels/fa_q8_0.metal index 146bbb31c526..bb29a262c1ea 100644 --- a/ggml/src/ggml-metal/kernels/fa_q8_0.metal +++ b/ggml/src/ggml-metal/kernels/fa_q8_0.metal @@ -41,6 +41,7 @@ template [[host_name("kernel_flash_attn_ext_q8_0_dk96_dv96" )]] kernel flash_at template [[host_name("kernel_flash_attn_ext_q8_0_dk96_dv64" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q8_0_dk112_dv112")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q8_0_dk128_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; +template [[host_name("kernel_flash_attn_ext_q8_0_dk128_dv96" )]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q8_0_dk192_dv192")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q8_0_dk192_dv128")]] kernel flash_attn_ext_t kernel_flash_attn_ext; template [[host_name("kernel_flash_attn_ext_q8_0_dk256_dv256")]] kernel flash_attn_ext_t kernel_flash_attn_ext; diff --git a/ggml/src/ggml-metal/kernels/fa_vec_f16.metal b/ggml/src/ggml-metal/kernels/fa_vec_f16.metal index 56b5adb840cd..bea54f9f73fd 100644 --- a/ggml/src/ggml-metal/kernels/fa_vec_f16.metal +++ b/ggml/src/ggml-metal/kernels/fa_vec_f16.metal @@ -56,8 +56,12 @@ template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128_q2_ne4")]] kerne template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f16_dk128_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; #if defined(GGML_METAL_HAS_BF16) template [[host_name("kernel_flash_attn_ext_vec_bf16_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_bf16_dk128_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; #endif template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f16_dk192_dv192_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; diff --git a/ggml/src/ggml-metal/kernels/fa_vec_f32.metal b/ggml/src/ggml-metal/kernels/fa_vec_f32.metal index 1c6bed0c61cf..18f862966a78 100644 --- a/ggml/src/ggml-metal/kernels/fa_vec_f32.metal +++ b/ggml/src/ggml-metal/kernels/fa_vec_f32.metal @@ -25,6 +25,7 @@ template [[host_name("kernel_flash_attn_ext_vec_f32_dk64_dv64")]] kernel flas template [[host_name("kernel_flash_attn_ext_vec_f32_dk96_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f32_dk96_dv64")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f32_dk128_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_f32_dk128_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f32_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f32_dk192_dv128")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_f32_dk256_dv256")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; diff --git a/ggml/src/ggml-metal/kernels/fa_vec_q4_0.metal b/ggml/src/ggml-metal/kernels/fa_vec_q4_0.metal index 7bb99ffd724c..6bf5fad75cf1 100644 --- a/ggml/src/ggml-metal/kernels/fa_vec_q4_0.metal +++ b/ggml/src/ggml-metal/kernels/fa_vec_q4_0.metal @@ -44,6 +44,9 @@ template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv128_q2_ne4")]] kern template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv128_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk128_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv192_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q4_0_dk192_dv192_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; diff --git a/ggml/src/ggml-metal/kernels/fa_vec_q4_1.metal b/ggml/src/ggml-metal/kernels/fa_vec_q4_1.metal index 8a6b08bca3fb..37277ee10203 100644 --- a/ggml/src/ggml-metal/kernels/fa_vec_q4_1.metal +++ b/ggml/src/ggml-metal/kernels/fa_vec_q4_1.metal @@ -44,6 +44,9 @@ template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv128_q2_ne4")]] kern template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv128_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk128_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv192_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q4_1_dk192_dv192_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; diff --git a/ggml/src/ggml-metal/kernels/fa_vec_q5_0.metal b/ggml/src/ggml-metal/kernels/fa_vec_q5_0.metal index 1633c4764a8a..26868a228a40 100644 --- a/ggml/src/ggml-metal/kernels/fa_vec_q5_0.metal +++ b/ggml/src/ggml-metal/kernels/fa_vec_q5_0.metal @@ -44,6 +44,9 @@ template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv128_q2_ne4")]] kern template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv128_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk128_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv192_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_0_dk192_dv192_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; diff --git a/ggml/src/ggml-metal/kernels/fa_vec_q5_1.metal b/ggml/src/ggml-metal/kernels/fa_vec_q5_1.metal index 2e7db017e621..9d39dc58a8d5 100644 --- a/ggml/src/ggml-metal/kernels/fa_vec_q5_1.metal +++ b/ggml/src/ggml-metal/kernels/fa_vec_q5_1.metal @@ -44,6 +44,9 @@ template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv128_q2_ne4")]] kern template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv128_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk128_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv192_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q5_1_dk192_dv192_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; diff --git a/ggml/src/ggml-metal/kernels/fa_vec_q8_0.metal b/ggml/src/ggml-metal/kernels/fa_vec_q8_0.metal index e9c6b152209b..7686ccc00126 100644 --- a/ggml/src/ggml-metal/kernels/fa_vec_q8_0.metal +++ b/ggml/src/ggml-metal/kernels/fa_vec_q8_0.metal @@ -44,6 +44,9 @@ template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv128_q2_ne4")]] kern template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv128_q4_ne1")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv128_q4_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv128_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv96")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv96_q2_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; +template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk128_dv96_q4_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv192")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv192_q1_ne4")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; template [[host_name("kernel_flash_attn_ext_vec_q8_0_dk192_dv192_q2_ne2")]] kernel flash_attn_ext_vec_t kernel_flash_attn_ext_vec; From 2c564f43dfdb0565745dc71cc7748d2a1305246a Mon Sep 17 00:00:00 2001 From: Pascal Date: Fri, 9 Oct 2026 10:29:22 +0200 Subject: [PATCH 005/431] ci: fix HIP quality check by ignoring the 320/256 FA kernel spill (#30212) --- scripts/hip/gcn-cdna-vgpr-check.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/hip/gcn-cdna-vgpr-check.py b/scripts/hip/gcn-cdna-vgpr-check.py index 6461fccd510b..1f1c2c903bc7 100644 --- a/scripts/hip/gcn-cdna-vgpr-check.py +++ b/scripts/hip/gcn-cdna-vgpr-check.py @@ -83,6 +83,7 @@ def main(): '_ZL18flash_attn_ext_f16ILi576ELi512ELi2ELi16ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', '_ZL18flash_attn_ext_f16ILi576ELi512ELi8ELi4ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', '_ZL18flash_attn_ext_vecILi256ELi1EL9ggml_type2ELS0_2ELb0EEvPKcS2_S2_S2_S2_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS6_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi320ELi256ELi2ELi32ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', } functions = parse_log_file(log_file) From b013e56a7147dbff5ec03175828442a94086507c Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Fri, 9 Oct 2026 10:34:45 +0200 Subject: [PATCH 006/431] ui : add the models manager (#29583) * ui : add the drawer, sheet and grouped list primitives Add the drawer and sheet overlay components, the toggle and toggle group, the shared searchable input, the collapsible sections and the grouped list, and the near viewport helper they position with. Assisted-by: pi:llama.cpp/DeepSeek-V4.1-Flash * ui : add the shared model components and data layer Add the model row pieces shared by the selector and the manager, the models store with its download status feed, the huggingface and migration services, and the model utils, enums and constants they read through. Assisted-by: pi:llama.cpp/DeepSeek-V4.1-Flash * ui : rework the models selector around its providers Group the selector components under their own folder, rework the list around the provider grouping, add the mobile trigger and the download item, and derive the selection state in one hook. Assisted-by: pi:llama.cpp/DeepSeek-V4.1-Flash * ui : add the models manager Add the manager table with its repo, quant and status rows, the filters and the toolbar, the row actions in a drawer, the downloads section, the manage models dialog and the stories and tests. Assisted-by: pi:llama.cpp/DeepSeek-V4.1-Flash * ui : rework the chat form actions and the mobile experience Move the add actions into a drawer and a sheet, float the model actions in a bar on a phone, open the context panel in a drawer, and derive the attachment, reasoning and tools menus in hooks. Assisted-by: pi:llama.cpp/DeepSeek-V4.1-Flash * ui : move the mcp servers to the sidebar rail and tidy the dialogs Move the mcp servers dialog to the sidebar rail with its menu entries, even out the dialogs on a phone, and group the mcp and settings stores behind their own modules. Assisted-by: pi:llama.cpp/DeepSeek-V4.1-Flash * ui : polish the shell The secondary button gets its own look back and the chat add button its own light surface. Pressable elements get a pointer cursor again, the font rendering smooths on the app shell, and the agent skills stay out of prettier's way. The root layout props probe rework that came with this polish reads the providers api url and stays with the providers change instead. Assisted-by: pi:llama.cpp/DeepSeek-V4.1-Flash * ui : wrap the model id classes Assisted-by: pi:llama.cpp/DeepSeek-V4.1-Flash * ui : load the model the new chat CTA picks Start a new chat selected the model but left it unloaded, so the chat opened against a server with nothing in memory. Both CTAs share a load helper now. Assisted-by: pi:llama.cpp/DeepSeek-V4.1-Flash * refactor: Post-review fixes --- tools/ui/.prettierignore | 4 + tools/ui/README.md | 160 ++++--- tools/ui/package-lock.json | 64 ++- tools/ui/package.json | 3 +- tools/ui/src/app.css | 29 +- .../app/chat/ChatForm/ChatForm.svelte | 23 +- .../ChatFormActionAddButton.svelte | 4 +- .../ChatFormActionAddDrawer.svelte | 206 +++++++++ .../ChatFormActionAddDropdown.svelte | 49 +- .../ChatFormActionAddReasoningSubmenu.svelte | 75 ---- .../ChatFormActionAddSheet.svelte | 284 ------------ .../ChatFormActionAddToolsSubmenu.svelte | 4 +- .../ChatFormActionsAdd.svelte | 6 +- .../ChatFormActionModels.svelte | 170 ------- .../ChatFormActionReasoning.svelte | 147 ++++++ .../ChatFormActionRecord.svelte | 2 +- .../ChatFormActions/ChatFormActions.svelte | 47 +- .../ChatFormContextGauge.svelte | 11 +- .../ContextGaugeDetailRow.svelte | 4 +- .../ContextGaugeDetails.svelte | 29 +- .../ContextGaugeDial.svelte | 5 +- .../ContextGaugePopup.svelte | 129 +++--- .../gauge-popup.svelte.ts | 2 +- .../ChatFormPickerCommand.svelte | 12 +- .../ChatMessageAssistantModel.svelte | 18 +- ...atMessageToolCallBlockSearchResults.svelte | 3 +- .../ChatMessageStatistics.svelte | 2 +- .../app/chat/ChatScreen/ChatScreenForm.svelte | 1 + .../ChatScreen/ChatScreenServerError.svelte | 2 +- tools/ui/src/lib/components/app/chat/index.ts | 17 +- .../app/dialogs/DialogManageModels.svelte | 40 ++ .../dialogs/DialogMcpResourcePreview.svelte | 7 +- .../dialogs/DialogMcpResourcesBrowser.svelte | 22 +- .../app/dialogs/DialogMcpServerAddNew.svelte | 7 +- .../app/dialogs/DialogMcpServers.svelte | 4 +- .../app/dialogs/DialogModelInformation.svelte | 408 +++-------------- .../app/dialogs/DialogSettingsChat.svelte | 18 +- .../src/lib/components/app/dialogs/index.ts | 43 +- .../components/app/forms/SearchInput.svelte | 5 +- .../app/misc/CollapsibleRegion.svelte | 83 ++++ .../app/misc/CollapsibleSection.svelte | 111 +++++ .../components/app/misc/GroupedList.svelte | 188 ++++++++ tools/ui/src/lib/components/app/misc/index.ts | 9 + .../components/app/models/ModelAvatar.svelte | 106 +++++ .../components/app/models/ModelBadge.svelte | 4 +- .../app/models/ModelCapabilities.svelte | 63 +++ .../app/models/ModelCapabilityIcons.svelte | 104 ++--- .../components/app/models/ModelContext.svelte | 58 +++ ...svelte => ModelDownloadProgressBar.svelte} | 4 +- .../app/models/ModelDraftSidecars.svelte | 42 ++ .../lib/components/app/models/ModelId.svelte | 47 +- .../app/models/ModelLoadControl.svelte | 107 +++++ .../app/models/ModelOrgAvatar.svelte | 130 ++++++ .../app/models/ModelRowActions.svelte | 75 ++++ .../models/ModelsManager/ModelsManager.svelte | 357 +++++++++++++++ .../ModelsManagerDownloadControl.svelte | 50 +++ .../ModelsManager/ModelsManagerFilters.svelte | 135 ++++++ .../ModelsManagerModelConfiguration.svelte | 110 +++++ ...elsManagerModelConfigurationActions.svelte | 85 ++++ ...delsManagerModelConfigurationHeader.svelte | 111 +++++ ...anagerModelConfigurationInformation.svelte | 197 ++++++++ .../ModelsManagerModelRow.svelte | 104 +++++ .../ModelsManagerModelsTable.svelte | 422 ++++++++++++++++++ .../ModelsManagerQuantRow.svelte | 71 +++ .../ModelsManager/ModelsManagerRepoRow.svelte | 95 ++++ .../ModelsManagerStatusCell.svelte | 43 ++ .../ModelsManagerTableToolbar.svelte | 88 ++++ .../app/models/ModelsManager/row-actions.ts | 61 +++ .../app/models/ModelsManager/utils.ts | 202 +++++++++ .../app/models/ModelsSection.svelte | 97 ++++ .../ModelsSelector/ModelsSelector.svelte | 405 +++++++++++++++++ .../ModelsSelectorDownloadItem.svelte | 130 ++++++ .../ModelsSelectorDrawer.svelte | 53 +++ .../ModelsSelectorDropdown.svelte | 248 ++++++++++ .../ModelsSelector/ModelsSelectorList.svelte | 235 ++++++++++ .../ModelsSelectorOption.svelte | 105 +++++ .../ModelsSelectorTriggerIcon.svelte | 23 + .../app/models/ModelsSelector/index.ts | 10 + .../app/models/ModelsSelectorDropdown.svelte | 417 ----------------- .../app/models/ModelsSelectorList.svelte | 76 ---- .../app/models/ModelsSelectorOption.svelte | 216 --------- .../app/models/ModelsSelectorSheet.svelte | 210 --------- .../discover/ModelsDiscoverAvatar.svelte | 109 ----- .../ui/src/lib/components/app/models/index.ts | 84 +--- .../app/navigation/DropdownMenuActions.svelte | 159 +++++-- .../navigation/DropdownMenuSearchable.svelte | 47 +- .../SidebarNavigation.svelte | 27 +- .../SidebarNavigationActions.svelte | 82 ++-- .../lib/components/ui/button/button.svelte | 4 +- .../ui/dialog/dialog-content.svelte | 4 +- .../components/ui/dialog/dialog-header.svelte | 3 +- .../components/ui/drawer/drawer-close.svelte | 7 + .../ui/drawer/drawer-content.svelte | 38 ++ .../ui/drawer/drawer-description.svelte | 17 + .../drawer-footer.svelte} | 4 +- .../drawer-header.svelte} | 7 +- .../components/ui/drawer/drawer-nested.svelte | 12 + .../ui/drawer/drawer-overlay.svelte | 20 + .../components/ui/drawer/drawer-portal.svelte | 7 + .../components/ui/drawer/drawer-title.svelte | 17 + .../ui/drawer/drawer-trigger.svelte | 7 + .../lib/components/ui/drawer/drawer.svelte | 12 + .../ui/src/lib/components/ui/drawer/index.ts | 38 ++ .../dropdown-menu-content.svelte | 2 +- tools/ui/src/lib/components/ui/sheet/index.ts | 36 -- .../components/ui/sheet/sheet-close.svelte | 7 - .../components/ui/sheet/sheet-content.svelte | 64 --- .../ui/sheet/sheet-description.svelte | 17 - .../components/ui/sheet/sheet-overlay.svelte | 20 - .../components/ui/sheet/sheet-title.svelte | 17 - .../components/ui/sheet/sheet-trigger.svelte | 7 - .../lib/components/ui/toggle-group/index.ts | 10 + .../ui/toggle-group/toggle-group-item.svelte | 35 ++ .../ui/toggle-group/toggle-group.svelte | 75 ++++ .../ui/src/lib/components/ui/toggle/index.ts | 13 + .../lib/components/ui/toggle/toggle.svelte | 54 +++ .../constants/attachment-menu.constants.ts | 17 + .../ui/src/lib/constants/backend.constants.ts | 2 + .../lib/constants/css-classes.constants.ts | 54 +++ tools/ui/src/lib/constants/icons.constants.ts | 59 ++- tools/ui/src/lib/constants/index.ts | 2 + .../lib/constants/model-loading.constants.ts | 22 + .../lib/constants/models-manager.constants.ts | 18 + .../lib/constants/settings-keys.constants.ts | 7 +- .../src/lib/constants/settings.constants.ts | 29 +- .../ui/src/lib/constants/storage.constants.ts | 9 + tools/ui/src/lib/constants/ui.constants.ts | 27 +- tools/ui/src/lib/enums/index.ts | 9 +- tools/ui/src/lib/enums/model.enums.ts | 28 ++ tools/ui/src/lib/enums/ui.enums.ts | 2 + .../lib/hooks/use-attachment-menu.svelte.ts | 2 +- .../lib/hooks/use-chat-form-model.svelte.ts | 121 +++++ .../lib/hooks/use-models-selector.svelte.ts | 140 +++--- .../lib/hooks/use-reasoning-menu.svelte.ts | 4 +- .../src/lib/hooks/use-tools-panel.svelte.ts | 2 +- .../src/lib/services/huggingface.service.ts | 131 +++++- .../ui/src/lib/services/migration.service.ts | 34 ++ tools/ui/src/lib/stores/mcp/index.svelte.ts | 14 +- .../ui/src/lib/stores/models/index.svelte.ts | 302 ++++++++++++- .../ui/src/lib/stores/models/status.svelte.ts | 73 ++- .../src/lib/stores/settings/index.svelte.ts | 2 +- tools/ui/src/lib/stores/ui.svelte.ts | 27 ++ tools/ui/src/lib/types/chat.d.ts | 5 +- tools/ui/src/lib/types/huggingface.d.ts | 7 + tools/ui/src/lib/types/index.ts | 1 + tools/ui/src/lib/types/models.d.ts | 32 ++ tools/ui/src/lib/utils/cors-proxy.ts | 15 + tools/ui/src/lib/utils/formatters.ts | 19 + tools/ui/src/lib/utils/index.ts | 19 +- tools/ui/src/lib/utils/model-families.ts | 47 ++ .../models/utils.ts => utils/model-list.ts} | 78 ++-- tools/ui/src/lib/utils/model-names.ts | 16 + tools/ui/src/lib/utils/near-viewport.ts | 34 ++ .../components/DialogPanelWrapper.svelte | 12 + .../components/ModelsManagerRowWrapper.svelte | 24 + .../components/ModelsManagerWrapper.svelte | 9 + .../ModelsSelectorHookHarness.svelte | 36 ++ .../context-panel-mobile.svelte.test.ts | 42 ++ .../client/dialog-focus-mobile.svelte.test.ts | 30 ++ .../client/mcp-icon-proxy-url.svelte.test.ts | 66 +++ .../client/model-avatar-orgs.svelte.test.ts | 60 +++ .../client/models-manager-rows.svelte.test.ts | 91 ++++ .../models-manager-utils.svelte.test.ts | 164 +++++++ .../models-selector-hidden.svelte.test.ts | 77 ++++ .../models-selector-mobile.svelte.test.ts | 56 +++ .../reasoning-control-mobile.svelte.test.ts | 41 ++ ...odel-org-name-key-migration.svelte.test.ts | 59 +++ .../stories/ModelsManager.stories.svelte | 110 +++++ .../stories/ModelsSelector.stories.svelte | 45 +- .../a11y/ChatScreenForm.a11y.stories.svelte | 21 +- .../a11y/ModelsManager.a11y.stories.svelte | 52 +++ tools/ui/tests/unit/formatters.test.ts | 24 + tools/ui/tests/unit/model-families.test.ts | 54 +++ tools/ui/tests/unit/model-list.test.ts | 104 +++++ 174 files changed, 8361 insertions(+), 2841 deletions(-) create mode 100644 tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDrawer.svelte delete mode 100644 tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddReasoningSubmenu.svelte delete mode 100644 tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddSheet.svelte delete mode 100644 tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte create mode 100644 tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionReasoning.svelte create mode 100644 tools/ui/src/lib/components/app/dialogs/DialogManageModels.svelte create mode 100644 tools/ui/src/lib/components/app/misc/CollapsibleRegion.svelte create mode 100644 tools/ui/src/lib/components/app/misc/CollapsibleSection.svelte create mode 100644 tools/ui/src/lib/components/app/misc/GroupedList.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelAvatar.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelCapabilities.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelContext.svelte rename tools/ui/src/lib/components/app/models/{discover/DownloadProgressBar.svelte => ModelDownloadProgressBar.svelte} (80%) create mode 100644 tools/ui/src/lib/components/app/models/ModelDraftSidecars.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelLoadControl.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelOrgAvatar.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelRowActions.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelsManager/ModelsManager.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerDownloadControl.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerFilters.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfiguration.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationActions.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationHeader.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationInformation.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerQuantRow.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerRepoRow.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerStatusCell.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerTableToolbar.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelsManager/row-actions.ts create mode 100644 tools/ui/src/lib/components/app/models/ModelsManager/utils.ts create mode 100644 tools/ui/src/lib/components/app/models/ModelsSection.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelector.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDownloadItem.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDrawer.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDropdown.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorList.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorOption.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorTriggerIcon.svelte create mode 100644 tools/ui/src/lib/components/app/models/ModelsSelector/index.ts delete mode 100644 tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte delete mode 100644 tools/ui/src/lib/components/app/models/ModelsSelectorList.svelte delete mode 100644 tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte delete mode 100644 tools/ui/src/lib/components/app/models/ModelsSelectorSheet.svelte delete mode 100644 tools/ui/src/lib/components/app/models/discover/ModelsDiscoverAvatar.svelte create mode 100644 tools/ui/src/lib/components/ui/drawer/drawer-close.svelte create mode 100644 tools/ui/src/lib/components/ui/drawer/drawer-content.svelte create mode 100644 tools/ui/src/lib/components/ui/drawer/drawer-description.svelte rename tools/ui/src/lib/components/ui/{sheet/sheet-header.svelte => drawer/drawer-footer.svelte} (81%) rename tools/ui/src/lib/components/ui/{sheet/sheet-footer.svelte => drawer/drawer-header.svelte} (60%) create mode 100644 tools/ui/src/lib/components/ui/drawer/drawer-nested.svelte create mode 100644 tools/ui/src/lib/components/ui/drawer/drawer-overlay.svelte create mode 100644 tools/ui/src/lib/components/ui/drawer/drawer-portal.svelte create mode 100644 tools/ui/src/lib/components/ui/drawer/drawer-title.svelte create mode 100644 tools/ui/src/lib/components/ui/drawer/drawer-trigger.svelte create mode 100644 tools/ui/src/lib/components/ui/drawer/drawer.svelte create mode 100644 tools/ui/src/lib/components/ui/drawer/index.ts delete mode 100644 tools/ui/src/lib/components/ui/sheet/index.ts delete mode 100644 tools/ui/src/lib/components/ui/sheet/sheet-close.svelte delete mode 100644 tools/ui/src/lib/components/ui/sheet/sheet-content.svelte delete mode 100644 tools/ui/src/lib/components/ui/sheet/sheet-description.svelte delete mode 100644 tools/ui/src/lib/components/ui/sheet/sheet-overlay.svelte delete mode 100644 tools/ui/src/lib/components/ui/sheet/sheet-title.svelte delete mode 100644 tools/ui/src/lib/components/ui/sheet/sheet-trigger.svelte create mode 100644 tools/ui/src/lib/components/ui/toggle-group/index.ts create mode 100644 tools/ui/src/lib/components/ui/toggle-group/toggle-group-item.svelte create mode 100644 tools/ui/src/lib/components/ui/toggle-group/toggle-group.svelte create mode 100644 tools/ui/src/lib/components/ui/toggle/index.ts create mode 100644 tools/ui/src/lib/components/ui/toggle/toggle.svelte create mode 100644 tools/ui/src/lib/constants/backend.constants.ts create mode 100644 tools/ui/src/lib/constants/models-manager.constants.ts create mode 100644 tools/ui/src/lib/hooks/use-chat-form-model.svelte.ts create mode 100644 tools/ui/src/lib/utils/model-families.ts rename tools/ui/src/lib/{components/app/models/utils.ts => utils/model-list.ts} (54%) create mode 100644 tools/ui/src/lib/utils/near-viewport.ts create mode 100644 tools/ui/tests/client/components/DialogPanelWrapper.svelte create mode 100644 tools/ui/tests/client/components/ModelsManagerRowWrapper.svelte create mode 100644 tools/ui/tests/client/components/ModelsManagerWrapper.svelte create mode 100644 tools/ui/tests/client/components/ModelsSelectorHookHarness.svelte create mode 100644 tools/ui/tests/client/context-panel-mobile.svelte.test.ts create mode 100644 tools/ui/tests/client/dialog-focus-mobile.svelte.test.ts create mode 100644 tools/ui/tests/client/mcp-icon-proxy-url.svelte.test.ts create mode 100644 tools/ui/tests/client/model-avatar-orgs.svelte.test.ts create mode 100644 tools/ui/tests/client/models-manager-rows.svelte.test.ts create mode 100644 tools/ui/tests/client/models-manager-utils.svelte.test.ts create mode 100644 tools/ui/tests/client/models-selector-hidden.svelte.test.ts create mode 100644 tools/ui/tests/client/models-selector-mobile.svelte.test.ts create mode 100644 tools/ui/tests/client/reasoning-control-mobile.svelte.test.ts create mode 100644 tools/ui/tests/client/settings-model-org-name-key-migration.svelte.test.ts create mode 100644 tools/ui/tests/stories/ModelsManager.stories.svelte create mode 100644 tools/ui/tests/stories/a11y/ModelsManager.a11y.stories.svelte create mode 100644 tools/ui/tests/unit/formatters.test.ts create mode 100644 tools/ui/tests/unit/model-families.test.ts create mode 100644 tools/ui/tests/unit/model-list.test.ts diff --git a/tools/ui/.prettierignore b/tools/ui/.prettierignore index 635cf99c7e9c..ba47fbe165fa 100644 --- a/tools/ui/.prettierignore +++ b/tools/ui/.prettierignore @@ -19,3 +19,7 @@ test-results # Vendored third party sources, kept byte identical to upstream src/lib/vendors/ + +# Agent skills installed from upstream, kept as installed +.agents/ +skills-lock.json diff --git a/tools/ui/README.md b/tools/ui/README.md index 99abfaa41fcf..04a6ea01657b 100644 --- a/tools/ui/README.md +++ b/tools/ui/README.md @@ -54,10 +54,13 @@ Llama UI supports two server operation modes: ### Multi-Model Support (ROUTER mode) +- **Models manager** - one table for every model the server can serve, split into loaded, downloading, favorites, local and hidden sections, with search, context/capability/modality filters, sorting, load/unload, delete from disk and hide. It opens from the sidebar, from a model row or from the selector. - **Model selector** with Loaded/Available groups - **Automatic loading** - Models load on selection - **Modality validation** - Prevents sending images to non-vision models - **LRU unloading** - Server auto-manages model cache +- **Downloads** - In-flight downloads stay visible above the selector and can be paused, resumed or cancelled from the manager +- **Hub metadata** - Avatars, context length and chat template come from the Hugging Face Hub only while the `Use Hugging Face Hub API for models metadata` setting is on. It is off by default, and the UI then shows what the server reports for `/v1/models` ### Keyboard Shortcuts @@ -144,6 +147,7 @@ proxy: { | ----------------- | ------------------------------- | -------------------------------------------------------- | | **Framework** | SvelteKit + Svelte 5 | Reactive UI with runes (`$state`, `$derived`, `$effect`) | | **UI Components** | shadcn-svelte + bits-ui | Accessible, customizable component library | +| **Drawers** | vaul-svelte | Draggable drawer overlays (mobile pickers, model pane) | | **Styling** | TailwindCSS 4 | Utility-first CSS with design tokens | | **Database** | IndexedDB (Dexie) | Client-side storage for conversations and messages | | **Build** | Vite | Fast bundling with static adapter | @@ -350,94 +354,103 @@ Components are organized in `app/` (application-specific) and `ui/` (shadcn-svel **Dialog Components** (`app/dialogs/`): -| Component | Responsibility | -| ------------------------------- | -------------------------------------------------------- | -| `DialogChatSettings` | Full-screen settings configuration | -| `DialogModelInformation` | Model details (context size, modalities, parallel slots) | -| `DialogChatAttachmentPreview` | Full preview for images, PDFs (text or page view), code | -| `DialogConfirmation` | Generic confirmation for destructive actions | -| `DialogConversationTitleUpdate` | Edit conversation title | +| Component | Responsibility | +| ------------------------------ | -------------------------------------------------------- | +| `DialogChatSettings` | Full-screen settings configuration | +| `DialogManageModels` | Models manager: browse, load, download and delete models | +| `DialogModelInformation` | Model details (context size, modalities, parallel slots) | +| `DialogChatAttachmentsPreview` | Full preview for images, PDFs (text or page view), code | +| `DialogConfirmation` | Generic confirmation for destructive actions | +| `DialogConversationRename` | Edit conversation title | **Server/Model Components** (`app/server/`, `app/models/`): -| Component | Responsibility | -| ------------------- | --------------------------------------------------------- | -| `ServerErrorSplash` | Error display when server is unreachable | -| `ModelsSelector` | Model dropdown with Loaded/Available groups (ROUTER mode) | +| Component | Responsibility | +| ----------------------------------- | ------------------------------------------------------------------------------------------------------------- | +| `ServerErrorSplash` | Error display when server is unreachable | +| `ModelsSelector/` | Model dropdown and mobile sheet: favorites, loaded and local sections, rows windowed and folded into families | +| `ModelsManager/` | Models manager table and model pane: sections, filters, sorting, quants folded per repo, downloads | +| `ModelAvatar` | Org avatar of a model, with the quantizer badge; follows the family grouping of the list | +| `ModelId`, `ModelBadge` | Model name, aliases, tags, quantization and draft sidecar badges | +| `ModelContext`, `ModelCapabilities` | Context window and capability icons, filled from the Hub when it is enabled | **Shared UI Components** (`app/misc/`): -| Component | Responsibility | -| -------------------------------- | ---------------------------------------------------------------- | -| `MarkdownContent` | Markdown rendering with KaTeX, syntax highlighting, copy buttons | -| `SyntaxHighlightedCode` | Code blocks with language detection and highlighting | -| `ActionButton`, `ActionDropdown` | Reusable action buttons and menus | -| `BadgeModality`, `BadgeInfo` | Status and capability badges | +| Component | Responsibility | +| ----------------------------------------- | ----------------------------------------------------------------------- | +| `MarkdownContent` | Markdown rendering with KaTeX, syntax highlighting, copy buttons | +| `SyntaxHighlightedCode` | Code blocks with language detection and highlighting | +| `ActionIcon`, `DropdownMenuActions` | Reusable action button and action menu | +| `BadgesModality`, `BadgeInfo` | Status and capability badges | +| `CollapsibleSection`, `CollapsibleRegion` | Section header with a trigger that expands in place | +| `GroupedList` | Grouped rows with foldable groups, per-group windows and show-more rows | +| `ScrollCarousel`, `TruncatedText` | Horizontal scroll strip and single-line truncation with tooltip | #### Hooks (`src/lib/hooks/`) Hooks are the thin view-layer between components and stores: they own UI concerns (scroll, drag-and-drop, keyboard shortcuts, pickers, selection) and translate store state into view state. -| Hook | Responsibility | -| ------------------------------- | -------------------------------------------------------------- | -| `use-chat-screen-active-model` | Active model resolution + modality capability detection | -| `use-processing-state` | View over `chatStore.processing` for streaming progress/tokens | -| `use-context-gauge` | View over `contextStatsStore` for the context usage gauge | -| `use-models-selector` | Model selector dropdown state (loaded/available groups) | -| `use-tools-panel` | Tools panel state | -| `use-reasoning-menu` | Reasoning-effort menu state | -| `use-attachment-menu` | Attachment menu + modality flags | -| `use-draft-messages` | Per-chat draft message/files persistence | -| `use-chat-form-pickers` | Chat form pickers (commands, mentions) | -| `use-debounced-search` | Shared debounced async search for pickers | -| `use-picker-navigation` | Picker keyboard navigation | -| `use-chat-message-edit-context` | Message edit context (content + extras) | -| `use-chat-screen-drag-and-drop` | Drag-and-drop state machine | -| `use-chat-screen-file-upload` | File upload queue + capability validation | -| `use-chat-screen-scroll` | Scroll container binding + navigation guard | -| `use-auto-scroll` | Auto-scroll controller for streaming | -| `use-marquee-selection` | Shift+click / marquee range selection | -| `use-keyboard-shortcuts` | Global keyboard shortcuts | -| `use-settings-navigation` | Settings section navigation | -| `use-pwa` | PWA install/update + version mismatch detection | +| Hook | Responsibility | +| ------------------------------- | ----------------------------------------------------------------------------------------------- | +| `use-chat-screen-active-model` | Active model resolution + modality capability detection | +| `use-processing-state` | View over `chatStore.processing` for streaming progress/tokens | +| `use-context-gauge` | View over `contextStatsStore` for the context usage gauge | +| `use-models-selector` | Model selector state: hidden models, family grouping, windowed rows, selection and load on pick | +| `use-tools-panel` | Tools panel state | +| `use-reasoning-menu` | Reasoning-effort menu state | +| `use-attachment-menu` | Attachment menu + modality flags | +| `use-draft-messages` | Per-chat draft message/files persistence | +| `use-chat-form-pickers` | Chat form pickers (commands, mentions) | +| `use-debounced-search` | Shared debounced async search for pickers | +| `use-picker-navigation` | Picker keyboard navigation | +| `use-chat-message-edit-context` | Message edit context (content + extras) | +| `use-chat-screen-drag-and-drop` | Drag-and-drop state machine | +| `use-chat-screen-file-upload` | File upload queue + capability validation | +| `use-chat-screen-scroll` | Scroll container binding + navigation guard | +| `use-auto-scroll` | Auto-scroll controller for streaming | +| `use-marquee-selection` | Shift+click / marquee range selection | +| `use-keyboard-shortcuts` | Global keyboard shortcuts | +| `use-settings-navigation` | Settings section navigation | +| `use-pwa` | PWA install/update + version mismatch detection | #### Stores (`src/lib/stores/`) Stores own reactive application state as Svelte 5 runes. Larger stores are split into directories and compose focused sub-stores behind a narrow host interface (see Architectural Patterns). -| Store | Responsibility | -| -------------------- | --------------------------------------------------------------------------------------------------------------- | -| `chatStore` | Chat lifecycle, streaming, abort control, error handling; composes `processing`, `activity`, `streams`, `flows` | -| `conversationsStore` | Conversation CRUD, message branching, navigation, import/export; composes `preferences` | -| `modelsStore` | Model list, selection, loading/unloading (ROUTER); composes `props`, `status` | -| `mcpStore` | MCP host role: multi-server lifecycle, tool routing; composes `health`, `resources` | -| `agenticStore` | Multi-turn agentic loop orchestration, tool execution; composes `gates` | -| `serverStore` | Server connection state, `/props`, role detection, modalities | -| `settingsStore` | User preferences, theme, parameter sync with server defaults | -| `toolsStore` | Tool registry: server + MCP tools, enabled set for the LLM | -| `permissionsStore` | Persisted tool permission grants | -| `contextStatsStore` | Context window usage for the active conversation | -| `draftMessagesStore` | Per-chat draft message/files | -| `deviceStore` | Browser environment signals (mobile, OS, theme) | -| `versionStore` | Build version information | +| Store | Responsibility | +| -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| `chatStore` | Chat lifecycle, streaming, abort control, error handling; composes `processing`, `activity`, `streams`, `flows` | +| `conversationsStore` | Conversation CRUD, message branching, navigation, import/export; composes `preferences` | +| `modelsStore` | Model list, selection, loading/unloading (ROUTER), recent picks, favorites, hidden models and list open state; composes `props`, `status` | +| `mcpStore` | MCP host role: multi-server lifecycle, tool routing, server icons; composes `health`, `resources` | +| `agenticStore` | Multi-turn agentic loop orchestration, tool execution; composes `gates` | +| `serverStore` | Server connection state, `/props`, role detection, modalities | +| `settingsStore` | User preferences, theme, parameter sync with server defaults | +| `toolsStore` | Tool registry: server + MCP tools, enabled set for the LLM | +| `permissionsStore` | Persisted tool permission grants | +| `contextStatsStore` | Context window usage for the active conversation | +| `draftMessagesStore` | Per-chat draft message/files | +| `deviceStore` | Browser environment signals (mobile, OS, theme) | +| `versionStore` | Build version information | #### Services (`src/lib/services/`) Services are a stateless protocol layer: static methods, pure I/O, no reactive state. Stores consume them for all API and storage access. -| Service | Responsibility | -| ----------------------------- | ------------------------------------------------------------------------- | -| `ChatService` | `/v1/chat/completions` streaming + SSE parsing, message format conversion | -| `ModelsService` | `/models`, `/models/load`, `/models/unload` | -| `PropsService` | `/props`, `/props?model=` | -| `DatabaseService` | IndexedDB operations via Dexie | -| `MCPService` | MCP protocol: transports, connect, list/execute tools, prompts, resources | -| `ToolsService` | Server tool list/execute/stream (`/tools`) | -| `SandboxService` | Browser JS execution in a sandboxed worker | -| `ParameterSyncService` | Syncs settings with server defaults | -| `ConversationTransferService` | Conversation import/export JSONL + ZIP format | -| `MigrationService` | Non-destructive localStorage/IndexedDB migrations | -| `RouterService` | Dynamic route URL construction | +| Service | Responsibility | +| ----------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| `ChatService` | `/v1/chat/completions` streaming + SSE parsing, message format conversion | +| `ModelsService` | `/models`, `/models/load`, `/models/unload` | +| `PropsService` | `/props`, `/props?model=` | +| `HuggingFaceService` | Hugging Face Hub metadata: model details, avatars, README and file tree, cached per repo and off unless the Hub setting is on | +| `DatabaseService` | IndexedDB operations via Dexie | +| `MCPService` | MCP protocol: transports, connect, list/execute tools, prompts, resources | +| `ToolsService` | Server tool list/execute/stream (`/tools`) | +| `SandboxService` | Browser JS execution in a sandboxed worker | +| `ParameterSyncService` | Syncs settings with server defaults | +| `ConversationTransferService` | Conversation import/export JSONL + ZIP format | +| `MigrationService` | Non-destructive localStorage/IndexedDB migrations | +| `RouterService` | Dynamic route URL construction | --- @@ -660,8 +673,17 @@ flowchart TB ``` - **IndexedDB**: Conversations and messages (large, structured data) -- **LocalStorage**: Settings, user parameter overrides, theme (small key-value data) -- **Memory only**: Server props, model list (fetched fresh on each session) +- **LocalStorage**: Settings, user parameter overrides, theme, model favorites, recent picks, hidden models and model list open state (small key-value data) +- **Memory only**: Server props, model list (fetched fresh on each session), Hugging Face Hub details (cached per repo for the session) + +### 9. Remote Assets Under Cross-Origin Isolation + +llama-server serves the UI with `Cross-Origin-Embedder-Policy: require-corp` and `Cross-Origin-Opener-Policy: same-origin`, so a cross-origin subresource has to be allowed by the target origin. The UI handles that per asset kind: + +- **Hub avatars** are requested in CORS mode (`crossorigin="anonymous"`), which the Hub allows +- **Hub metadata** is fetched with `fetch()`, which is a CORS request by default, and only while the Hub setting is on +- **MCP server icons and web-search result favicons** have no CORS guarantee, so they go through the server's `/cors-proxy` when the user started llama-server with `--ui-mcp-proxy`; without it they fall back to the bundled glyph +- **Per-row Hub lookups** wait until the row is near the viewport (the `nearViewport` action), so a long model list does not fire one request per row on mount --- diff --git a/tools/ui/package-lock.json b/tools/ui/package-lock.json index 86d599e671c5..1c0e4ec98abf 100644 --- a/tools/ui/package-lock.json +++ b/tools/ui/package-lock.json @@ -12,7 +12,7 @@ "@eslint/compat": "1.4.1", "@eslint/js": "9.39.2", "@internationalized/date": "3.12.2", - "@lucide/svelte": "1.25.0", + "@lucide/svelte": "1.46.0", "@modelcontextprotocol/sdk": "1.30.0", "@playwright/test": "1.56.1", "@storybook/addon-a11y": "10.5.6", @@ -79,6 +79,7 @@ "unified": "11.0.5", "unist-util-visit": "5.1.0", "uuid": "13.0.2", + "vaul-svelte": "1.0.0-next.7", "vite": "7.3.6", "vite-plugin-devtools-json": "0.2.1", "vitest": "4.1.10", @@ -3225,9 +3226,9 @@ } }, "node_modules/@lucide/svelte": { - "version": "1.25.0", - "resolved": "https://registry.npmjs.org/@lucide/svelte/-/svelte-1.25.0.tgz", - "integrity": "sha512-v9m+dD68jxVnqkU3K59mG/RSRFlPGzmKCGSyMfnXcaGv9jODDQMyQkcp1CGvk3Y/cUj9v7f8rw1n//K0B53xGQ==", + "version": "1.46.0", + "resolved": "https://registry.npmjs.org/@lucide/svelte/-/svelte-1.46.0.tgz", + "integrity": "sha512-Ru8jDjoCVLJ7J7pPRUdY+odSBV2BI6rj68JTmT+ENq/ZwYOPGPAiKVvKz+XLyAoK4HAwlRTzZHpsMvrHfsetqg==", "dev": true, "license": "ISC", "peerDependencies": { @@ -16998,6 +16999,61 @@ "node": ">= 0.8" } }, + "node_modules/vaul-svelte": { + "version": "1.0.0-next.7", + "resolved": "https://registry.npmjs.org/vaul-svelte/-/vaul-svelte-1.0.0-next.7.tgz", + "integrity": "sha512-7zN7Bi3dFQixvvbUJY9uGDe7Ws/dGZeBQR2pXdXmzQiakjrxBvWo0QrmsX3HK+VH+SZOltz378cmgmCS9f9rSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "runed": "^0.23.2", + "svelte-toolbelt": "^0.7.1" + }, + "engines": { + "node": ">=18", + "pnpm": ">=8.7.0" + }, + "peerDependencies": { + "svelte": "^5.0.0" + } + }, + "node_modules/vaul-svelte/node_modules/runed": { + "version": "0.23.4", + "resolved": "https://registry.npmjs.org/runed/-/runed-0.23.4.tgz", + "integrity": "sha512-9q8oUiBYeXIDLWNK5DfCWlkL0EW3oGbk845VdKlPeia28l751VpfesaB/+7pI6rnbx1I6rqoZ2fZxptOJLxILA==", + "dev": true, + "funding": [ + "https://github.com/sponsors/huntabyte", + "https://github.com/sponsors/tglide" + ], + "dependencies": { + "esm-env": "^1.0.0" + }, + "peerDependencies": { + "svelte": "^5.7.0" + } + }, + "node_modules/vaul-svelte/node_modules/svelte-toolbelt": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/svelte-toolbelt/-/svelte-toolbelt-0.7.1.tgz", + "integrity": "sha512-HcBOcR17Vx9bjaOceUvxkY3nGmbBmCBBbuWLLEWO6jtmWH8f/QoWmbyUfQZrpDINH39en1b8mptfPQT9VKQ1xQ==", + "dev": true, + "funding": [ + "https://github.com/sponsors/huntabyte" + ], + "dependencies": { + "clsx": "^2.1.1", + "runed": "^0.23.2", + "style-to-object": "^1.0.8" + }, + "engines": { + "node": ">=18", + "pnpm": ">=8.7.0" + }, + "peerDependencies": { + "svelte": "^5.0.0" + } + }, "node_modules/vfile": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/vfile/-/vfile-6.0.3.tgz", diff --git a/tools/ui/package.json b/tools/ui/package.json index 40176047417f..7b15f3343ff4 100644 --- a/tools/ui/package.json +++ b/tools/ui/package.json @@ -32,7 +32,7 @@ "@eslint/compat": "1.4.1", "@eslint/js": "9.39.2", "@internationalized/date": "3.12.2", - "@lucide/svelte": "1.25.0", + "@lucide/svelte": "1.46.0", "@modelcontextprotocol/sdk": "1.30.0", "@playwright/test": "1.56.1", "@storybook/addon-a11y": "10.5.6", @@ -99,6 +99,7 @@ "unified": "11.0.5", "unist-util-visit": "5.1.0", "uuid": "13.0.2", + "vaul-svelte": "1.0.0-next.7", "vite": "7.3.6", "vite-plugin-devtools-json": "0.2.1", "vitest": "4.1.10", diff --git a/tools/ui/src/app.css b/tools/ui/src/app.css index f9b544bebc56..ccf13fbec3fc 100644 --- a/tools/ui/src/app.css +++ b/tools/ui/src/app.css @@ -6,6 +6,33 @@ @custom-variant dark (&:is(.dark *)); +/* shadcn components key their orientation styles off these, toggle-group among them */ +@custom-variant data-horizontal (&[data-orientation='horizontal']); +@custom-variant data-vertical (&[data-orientation='vertical']); + +/* + * Tailwind v4 dropped the pointer cursor from buttons. Every control a user can + * press gets it back, and anything disabled keeps the cursor its own class asks for. + */ +@layer base { + button:not(:disabled), + [role='button']:not([data-disabled]), + [role='checkbox']:not([data-disabled]), + [role='combobox']:not([data-disabled]), + [role='menuitem']:not([data-disabled]), + [role='menuitemcheckbox']:not([data-disabled]), + [role='menuitemradio']:not([data-disabled]), + [role='option']:not([data-disabled]), + [role='radio']:not([data-disabled]), + [role='switch']:not([data-disabled]), + [role='tab']:not([data-disabled]), + label[for], + select, + summary { + cursor: pointer; + } +} + :root { --radius: 0.625rem; --background: oklch(1 0 0); @@ -140,7 +167,7 @@ } body { - @apply bg-background text-foreground; + @apply bg-background text-foreground antialiased; scrollbar-width: thin; scrollbar-gutter: stable; } diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte index 893f8077dd6a..397ddc322b1a 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte @@ -8,8 +8,7 @@ ChatFormInputFileInputInvisible, ChatFormMcpResourcesList, ChatFormPickers, - DialogMcpResourcesBrowser, - DialogMcpServers + DialogMcpResourcesBrowser } from '$lib/components/app'; import { CLIPBOARD_CONTENT_QUOTE_PREFIX, @@ -34,7 +33,8 @@ modelsStore, serverStore, settingsStore, - toolsStore + toolsStore, + uiStore } from '$lib/stores'; import type { FileMentionEntry, @@ -193,7 +193,6 @@ let preSelectedResourceUri = $state(undefined); // MCP Servers Dialog State - let isMcpServersDialogOpen = $state(false); let currentConfig = $derived(settingsStore.config); @@ -478,6 +477,19 @@ queueMicrotask(() => inputRef?.focus()); } + // A flow outside the chat, e.g. the models manager closing onto it, asks for the + // composer back. The dialog's focus scope is still tearing down, so focus on a + // frame of its own. + $effect(() => { + if (uiStore.manageModelsOpen) return; + + if (!uiStore.composerFocusRequested) return; + + uiStore.composerFocusRequested = false; + + setTimeout(() => inputRef?.focus(), 0); + }); + // Splice the mention link in place of the `@` token. Uses the // live cursor, not a stale snapshot - the token may have been edited. function handleMentionSelect(entry: FileMentionEntry) { @@ -628,7 +640,6 @@ isReasoning={chatStore.isReasoning} {isRecording} onFileUpload={handleFileUpload} - onMcpSettingsClick={() => (isMcpServersDialogOpen = true)} onMicClick={handleMicClick} {onStop} onSystemPromptClick={() => onSystemPromptClick?.({ files: uploadedFiles, message: value })} @@ -667,5 +678,3 @@ }} preSelectedUri={preSelectedResourceUri} /> - - diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddButton.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddButton.svelte index 60e7bd1d4aa2..443046615ba7 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddButton.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddButton.svelte @@ -15,11 +15,11 @@ + + + + {#if toolsPanel.totalToolCount > 0} + (toolsExpanded = open)} open={toolsExpanded}> + + {#if toolsExpanded} + + {:else} + + {/if} + + + + {ATTACHMENT_MENU_TEXT.TOOLS} + + + +
+ {#each toolsPanel.categoryGroups as group (group.key)} + {@render groupRow(group)} + {/each} + + {#each toolsPanel.mcpGroups as group (group.key)} + {@render groupRow(group)} + {/each} +
+ + + {/if} + + + + + +{#snippet groupRow(group: ToolGroup)} + {@const checkState = toolsPanel.getGroupCheckState(group)} + {@const enabledCount = toolsPanel.getEnabledToolCount(group)} + {@const favicon = toolsPanel.getFavicon(group)} + {@const groupDisabled = toolsPanel.isGroupDisabled(group)} + + +{/snippet} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte index 1ce731845fe7..2cae6e8caeba 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte @@ -1,23 +1,20 @@ - - - - {#if reasoning.isReasoningActive} - - {:else if reasoning.isOff} - - {:else} - - {/if} - - - Reasoning - - - {reasoning.currentEffort} - - - - - - {#each reasoning.levels as level (level.value)} - {@const tokenLabel = reasoning.tokenLabel(level)} - reasoning.select(level)} - > - {#if reasoning.isSelected(level)} - - {:else} -
- {/if} - - {level.label} - - {#if tokenLabel} - - {tokenLabel} - - {/if} - - {#if level.hasInfo} - - - - - - -

Maximum reasoning effort with extended context usage

- - - {/if} - - {/each} - - diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddSheet.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddSheet.svelte deleted file mode 100644 index acd0f4d21161..000000000000 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddSheet.svelte +++ /dev/null @@ -1,284 +0,0 @@ - - -
- - {@render trigger({ disabled: chatFormActions.disabled, onclick: () => (sheetOpen = true) })} - - - - Add to chat - - - Add files, system prompt or configure MCP servers - - - -
- {#if reasoning.modelSupportsThinking} - (reasoningExpanded = open)} - open={reasoningExpanded} - > - - {#if reasoningExpanded} - - {:else} - - {/if} - - {#if reasoning.thinkingEnabled} - - {:else if reasoning.isOff} - - {:else} - - {/if} - - Reasoning - - - {reasoning.currentEffort} - - - - -
- {#each reasoning.levels as level (level.value)} - {@const tokenLabel = reasoning.tokenLabel(level)} - - {/each} -
- - - {/if} - - (filesExpanded = open)} open={filesExpanded}> - - {#if filesExpanded} - - {:else} - - {/if} - - - - Add files - - - -
- {#each ATTACHMENT_FILE_ITEMS as item (item.id)} - {@const enabled = attachmentMenu.isItemEnabled(item.enabledWhen)} - {#if enabled} - - {:else if item.disabledTooltip} - - - - - - -

{item.disabledTooltip}

- - - {/if} - {/each} -
- - - - - - {#if toolsPanel.totalToolCount > 0} - (toolsExpanded = open)} open={toolsExpanded}> - - {#if toolsExpanded} - - {:else} - - {/if} - - - - Tools - - - {toolsPanel.totalToolCount} tool{toolsPanel.totalToolCount !== 1 ? 's' : ''} - - - - -
- {#each toolsPanel.categoryGroups as group (group.key)} - {@render sheetGroupRow(group)} - {/each} - - {#each toolsPanel.mcpGroups as group (group.key)} - {@render sheetGroupRow(group)} - {/each} -
- - - {/if} - - -
- - -
- -{#snippet sheetGroupRow(group: ToolGroup)} - {@const checkState = toolsPanel.getGroupCheckState(group)} - {@const enabledCount = toolsPanel.getEnabledToolCount(group)} - {@const favicon = toolsPanel.getFavicon(group)} - {@const groupDisabled = toolsPanel.isGroupDisabled(group)} - - -{/snippet} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddToolsSubmenu.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddToolsSubmenu.svelte index f49544171480..fa40339be2be 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddToolsSubmenu.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddToolsSubmenu.svelte @@ -4,7 +4,7 @@ import * as Collapsible from '$lib/components/ui/collapsible'; import * as DropdownMenu from '$lib/components/ui/dropdown-menu'; import * as Tooltip from '$lib/components/ui/tooltip'; - import { CLI_FLAGS, ICON_CLASS_DEFAULT } from '$lib/constants'; + import { ATTACHMENT_MENU_TEXT, CLI_FLAGS, ICON_CLASS_DEFAULT } from '$lib/constants'; import { useToolsPanel } from '$lib/hooks/use-tools-panel.svelte'; import { mcpStore, toolsStore } from '$lib/stores'; import type { ToolGroup } from '$lib/types'; @@ -17,7 +17,7 @@ - Tools + {ATTACHMENT_MENU_TEXT.TOOLS} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionsAdd.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionsAdd.svelte index b2581f11eed2..50eb9be1c017 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionsAdd.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionsAdd.svelte @@ -1,16 +1,16 @@ {#if deviceStore.isMobile} - + {#snippet trigger({ disabled, onclick })} {/snippet} - + {:else} {/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte deleted file mode 100644 index a4baa40bf994..000000000000 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte +++ /dev/null @@ -1,170 +0,0 @@ - - -{#if deviceStore.isMobile} - -{:else} - -{/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionReasoning.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionReasoning.svelte new file mode 100644 index 000000000000..29de5ec26366 --- /dev/null +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionReasoning.svelte @@ -0,0 +1,147 @@ + + +{#snippet triggerLabel()} + + {#if reasoning.isOff} + + {:else} + + {/if} + + {#if !isDefault} + {reasoning.currentEffort} + {/if} + +{/snippet} + +{#snippet levelRow(level: ReasoningEffortLevel)} + {@const tokenLabel = reasoning.tokenLabel(level)} + + +{/snippet} + + +{#if isMobile} + + + + + + Reasoning effort + + +
+ {#each reasoning.levels as level (level.value)} + {@render levelRow(level)} + {/each} +
+ + +{:else} + + + {#snippet child({ props })} + + {/snippet} + + + + {#each reasoning.levels as level (level.value)} + {@const tokenLabel = reasoning.tokenLabel(level)} + + reasoning.select(level)}> + {#if reasoning.isSelected(level)} + + {:else} +
+ {/if} + + {level.label} + + {#if tokenLabel} + {tokenLabel} + {/if} + + {#if level.hasInfo} + + + + + + +

{MAX_EFFORT_HINT}

+ + + {/if} + + {/each} + + +{/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte index d1dd3fe46c8b..7c305acbee9a 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte @@ -27,7 +27,7 @@ + {/snippet} + + {:else} + + {/if} +{/snippet} + +{#snippet region()} + + {@render children()} +{/snippet} + +{#if triggerPosition === 'top'} + {@render triggerButton()} + + {@render region()} +{:else} + {@render region()} + + {@render triggerButton()} +{/if} diff --git a/tools/ui/src/lib/components/app/misc/GroupedList.svelte b/tools/ui/src/lib/components/app/misc/GroupedList.svelte new file mode 100644 index 000000000000..e249918c507c --- /dev/null +++ b/tools/ui/src/lib/components/app/misc/GroupedList.svelte @@ -0,0 +1,188 @@ + + + + +{#if groups} + {#each windowed as entry (entry.key)} + {@const expanded = !collapsed.has(entry.key)} + + {#if group} +
+ {@render group({ + depth: 0, + expanded, + group: entry.group, + key: entry.key, + toggle: () => toggleGroup(entry.key) + })} +
+ {/if} + + + {#each entry.rows as row (keyOf(row))} + {@render item({ depth: 1, entry: row })} + {/each} + + {#if entry.hidden > 0 && more} + {@render more({ + count: Math.min(groupWindow, entry.hidden), + onMore: () => growGroup(entry.key), + unit: GroupedListUnit.ENTRIES + })} + {/if} + + {/each} + + {#if windowed.length < (groups?.length ?? 0) && more} + {@render more({ + count: groupPage.count, + onMore: () => (sectionSteps += Math.max(1, groupPage.weight)), + unit: GroupedListUnit.GROUPS + })} + {/if} +{:else} + {#each flatRows as row (keyOf(row))} + {@render item({ depth: 0, entry: row })} + {/each} + + {#if flatRows.length < items.length && more} + {@render more({ + count: Math.min(sectionWindow, items.length - flatRows.length), + onMore: () => (sectionSteps += Math.min(sectionWindow, items.length - flatRows.length)), + unit: GroupedListUnit.ENTRIES + })} + {/if} +{/if} diff --git a/tools/ui/src/lib/components/app/misc/index.ts b/tools/ui/src/lib/components/app/misc/index.ts index a10410ef9289..a4ee2ebc35c7 100644 --- a/tools/ui/src/lib/components/app/misc/index.ts +++ b/tools/ui/src/lib/components/app/misc/index.ts @@ -59,3 +59,12 @@ export { default as CodeBlockActions } from './CodeBlockActions.svelte'; * `class` and `style` props and inherits color via `currentColor`. */ export { default as Logo } from './Logo.svelte'; + +/** Section with a trigger that expands in place. */ +export { default as CollapsibleSection } from './CollapsibleSection.svelte'; +export { + default as GroupedList, + type GroupedListGroup, + GroupedListUnit +} from './GroupedList.svelte'; +export { default as CollapsibleRegion } from './CollapsibleRegion.svelte'; diff --git a/tools/ui/src/lib/components/app/models/ModelAvatar.svelte b/tools/ui/src/lib/components/app/models/ModelAvatar.svelte new file mode 100644 index 000000000000..5e1234a27267 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelAvatar.svelte @@ -0,0 +1,106 @@ + + +{#if orgName && hubEnabled} + (isNearViewport = true)} + class={['inline-flex shrink-0', className]} + > + + +{:else} + {@render fallback?.()} +{/if} diff --git a/tools/ui/src/lib/components/app/models/ModelBadge.svelte b/tools/ui/src/lib/components/app/models/ModelBadge.svelte index d5b723ff9039..53cdcc26a8f8 100644 --- a/tools/ui/src/lib/components/app/models/ModelBadge.svelte +++ b/tools/ui/src/lib/components/app/models/ModelBadge.svelte @@ -1,8 +1,8 @@ + + (isNearViewport = true)} class={className}> + + diff --git a/tools/ui/src/lib/components/app/models/ModelCapabilityIcons.svelte b/tools/ui/src/lib/components/app/models/ModelCapabilityIcons.svelte index c9505a95eb5e..3743a158fdca 100644 --- a/tools/ui/src/lib/components/app/models/ModelCapabilityIcons.svelte +++ b/tools/ui/src/lib/components/app/models/ModelCapabilityIcons.svelte @@ -1,6 +1,14 @@ - - - {#if supportsToolUse && !hideCapabilities} - - - - + let capabilities = $derived([ + ...(supportsToolUse ? [ModelCapability.TOOL_USE] : []), + ...(supportsThinking ? [ModelCapability.REASONING] : []) + ]); - -

Tool use

- - - {/if} + let shownModalities = $derived( + MODALITY_ORDER.filter((modality) => modalities?.[MODALITY_FLAG_KEYS[modality]]) + ); - {#if supportsThinking && !hideCapabilities} - - - - + // an icon-less box still takes the gap where its icons would sit, so a caller with + // nothing to show renders no box at all + let hasIcons = $derived( + (!hideCapabilities && capabilities.length > 0) || + (shownModalities.length > 0 && !hideModalities) + ); + - -

Reasoning

- - - {/if} +{#if hasIcons} + + {#if !hideCapabilities} + {#each capabilities as capability (capability)} + {@const Icon = CAPABILITY_ICONS[capability]} - {#if hasModalityIcons && !hideModalities} - - {#if modalities?.vision} - + -

Vision

+

{CAPABILITY_LABELS[capability]}

 - {/if} - - {#if modalities?.video} - - - + {/each} + {/if} - -

Video

- - - {/if} + {#if shownModalities.length > 0 && !hideModalities} + + {#each shownModalities as modality (modality)} + {@const Icon = MODALITY_ICONS[modality]} - {#if modalities?.audio} - - - - + + + + - -

Audio

- - - {/if} - - {/if} - + +

{MODALITY_LABELS[modality]}

+ + + {/each} + + {/if} + +{/if} diff --git a/tools/ui/src/lib/components/app/models/ModelContext.svelte b/tools/ui/src/lib/components/app/models/ModelContext.svelte new file mode 100644 index 000000000000..6d5395fc4497 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelContext.svelte @@ -0,0 +1,58 @@ + + + (isNearViewport = true)} + class={['text-sm text-muted-foreground', className]} +> + {formatContextLength(values)} + diff --git a/tools/ui/src/lib/components/app/models/discover/DownloadProgressBar.svelte b/tools/ui/src/lib/components/app/models/ModelDownloadProgressBar.svelte similarity index 80% rename from tools/ui/src/lib/components/app/models/discover/DownloadProgressBar.svelte rename to tools/ui/src/lib/components/app/models/ModelDownloadProgressBar.svelte index 6f4973c12ed1..c3a89c0ba45c 100644 --- a/tools/ui/src/lib/components/app/models/discover/DownloadProgressBar.svelte +++ b/tools/ui/src/lib/components/app/models/ModelDownloadProgressBar.svelte @@ -18,14 +18,14 @@ {#if overlay}
{:else}
diff --git a/tools/ui/src/lib/components/app/models/ModelDraftSidecars.svelte b/tools/ui/src/lib/components/app/models/ModelDraftSidecars.svelte new file mode 100644 index 000000000000..9b655f8432aa --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelDraftSidecars.svelte @@ -0,0 +1,42 @@ + + +{#each badges as badge (badge.kind + (badge.quant ?? ''))} + + + + {badge.kind} + +{/each} diff --git a/tools/ui/src/lib/components/app/models/ModelId.svelte b/tools/ui/src/lib/components/app/models/ModelId.svelte index ba3a976aef8b..26d72f662ff2 100644 --- a/tools/ui/src/lib/components/app/models/ModelId.svelte +++ b/tools/ui/src/lib/components/app/models/ModelId.svelte @@ -1,13 +1,15 @@ + +
+ {#if isLoading} + + {:else} + + {#if showAction && revealOnHover} + {#if isFailed} + + {:else} + + {/if} + {:else} + {#if isFailed} + + {:else} + + {/if} + {/if} + + +
+ {#if isFailed} + modelsStore.status.load(option.model)} + stopPropagationOnClick + tooltip="Retry loading model" + tooltipAsTitle + /> + {:else if isLoaded || isSleeping} + modelsStore.status.unload(option.model)} + stopPropagationOnClick + tooltip="Unload model" + tooltipAsTitle + /> + {:else} + modelsStore.status.load(option.model)} + stopPropagationOnClick + tooltip="Load model" + tooltipAsTitle + /> + {/if} +
+ {/if} +
diff --git a/tools/ui/src/lib/components/app/models/ModelOrgAvatar.svelte b/tools/ui/src/lib/components/app/models/ModelOrgAvatar.svelte new file mode 100644 index 000000000000..5befb9306d07 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelOrgAvatar.svelte @@ -0,0 +1,130 @@ + + +{#if hubEnabled} + + {#if orgAvatarFailed} + + {:else} +
+ + { + failedAvatarOrgs.set(org, Date.now()); + }} + src={HuggingFaceService.getAvatarUrl(org)} + /> +
+ {/if} + + {#if quantOrg && quantOrg !== org} + + + {#if quantAvatarFailed} + + {:else} + { + failedAvatarOrgs.set(quantOrg ?? '', Date.now()); + }} + src={HuggingFaceService.getAvatarUrl(quantOrg)} + /> + {/if} + + {/if} + +{/if} diff --git a/tools/ui/src/lib/components/app/models/ModelRowActions.svelte b/tools/ui/src/lib/components/app/models/ModelRowActions.svelte new file mode 100644 index 000000000000..8f173174aac3 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelRowActions.svelte @@ -0,0 +1,75 @@ + + +
event.stopPropagation()} + onkeydown={(event) => event.stopPropagation()} + role="presentation" +> + + // a phone has no manager: its information dialog takes the icon + deviceStore.isMobile + ? uiStore.openModelInformation(option) + : uiStore.openModelsManager(option.id)} + tooltip="Manage model" + tooltipAsTitle + /> + + {#if isFav} + + + modelsStore.toggleFavorite(option.model)} + tooltip="Remove from favorites" + tooltipAsTitle + /> + + + + + {:else} + modelsStore.toggleFavorite(option.model)} + tooltip="Add to favorites" + tooltipAsTitle + /> + {/if} +
diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManager.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManager.svelte new file mode 100644 index 000000000000..e6020e917202 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManager.svelte @@ -0,0 +1,357 @@ + + +
+
+ (selectedId = option.id)} + {selectedId} + {toolbarEnd} + /> +
+ + +
+ +
+ {#if shownOption} + {#key shownId} + (selectedId = null)} + onToggleLoad={() => void toggleLoad(shownOption)} + onUseInChat={() => void useInChat(shownOption)} + onUseInNewChat={() => void useInNewChat(shownOption)} + option={shownOption} + /> + {/key} + {/if} +
+
+
+ + diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerDownloadControl.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerDownloadControl.svelte new file mode 100644 index 000000000000..0e81500f2c1f --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerDownloadControl.svelte @@ -0,0 +1,50 @@ + + + +
+ + {percent !== null ? `${percent}%` : isPaused ? 'Paused' : ''} + + + +
diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerFilters.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerFilters.svelte new file mode 100644 index 000000000000..8b4c3bf37e62 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerFilters.svelte @@ -0,0 +1,135 @@ + + + + + (contextLimit = Number(value))} + type="single" + value={String(contextLimit)} + > + + Context: + + {contextLabel} + + + + {#each CONTEXT_STEPS as step (step.value)} + {step.label} + {/each} + + + + + {#each toggles as toggle (toggle.value)} + + + + {/each} + + diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfiguration.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfiguration.svelte new file mode 100644 index 000000000000..8a109a8db41e --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfiguration.svelte @@ -0,0 +1,110 @@ + + +
+ + +
+ + +
+ +
+
+ + + {#if deviceStore.isMobile} +
+ +
+ {/if} + + (deleteOpen = false)} + open={deleteOpen} + repoWithTag={option.model} + /> +
diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationActions.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationActions.svelte new file mode 100644 index 000000000000..b7fc10997504 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationActions.svelte @@ -0,0 +1,85 @@ + + +
+ {#if isMobile} + + + {/if} + +
+ {#if !isMobile} + + {/if} + + {#if canToggleLoad} + + {/if} +
+
diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationHeader.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationHeader.svelte new file mode 100644 index 000000000000..40f5d0d5481b --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationHeader.svelte @@ -0,0 +1,111 @@ + + + +
+
+
+ + + +
+ + + +

+ + + + {statusLabel} + +

+
+
+ + +
+ + + {#if !isMobile} + + {/if} +
diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationInformation.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationInformation.svelte new file mode 100644 index 000000000000..7eed08516af6 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationInformation.svelte @@ -0,0 +1,197 @@ + + +
+
+ Model + + + + {option.model} + + + + +
+ + {#each visibleRows as row (row.label)} +
+ {row.label} + + + {#if row.value === null} + - + {:else} + + {#if row.isBadge} + {row.value} + {:else} + {row.value} + {/if} + + + {#if row.isCopyable} + + {/if} + {/if} + +
+ {/each} + + {#if visibleRows.length === 0} +

+ The model reports its metadata once it is loaded. +

+ {/if} + + {#if modalities.length > 0} +
+ Modalities + + +
+ {/if} +
+ +{#if !serverProps} +

+ /props values - file path, slots, build info - appear once the model is loaded. Reading this + page never loads a model. +

+{/if} + +
+ + {#snippet trigger()} + Chat Template + {/snippet} + + 
{serverProps?.chat_template ??
+				gguf?.chat_template ??
+				'Shown once the model is loaded.'}
+ +
diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte new file mode 100644 index 000000000000..f4d86b7293aa --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte @@ -0,0 +1,104 @@ + + +
+ + + + + + + + {#if download} + + {/if} + +
+ +
+
diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte new file mode 100644 index 000000000000..2a3c2206b294 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelsTable.svelte @@ -0,0 +1,422 @@ + + +{#snippet sortHeader(key: ModelsTableSortKey, label: string)} + +{/snippet} + +{#snippet entryTree(entry: ModelQuantGroup, indent = 0)} + {#if entry.quants.length > 1} + toggleQuants(entry.key)} + /> + + + {#each entry.quants as quant (quant.id)} + + {/each} + + {:else} + + {/if} +{/snippet} + +{#snippet familyRow({ + expanded, + group: family, + toggle +}: { + expanded: boolean; + group: ModelFamilyGroup; + toggle: () => void; +})} + {@const countLabel = `${family.entries.length} model${family.entries.length === 1 ? '' : 's'}`} + +
handleFamilyKeydown(event, toggle)} + role="button" + tabindex="0" + > + + + + {family.label} + + {countLabel} + + + + + + + + + {#if expanded} + + {:else} + + {/if} + +
+{/snippet} + +{#snippet listItem({ depth, entry }: { depth: number; entry: ModelQuantGroup })} + {@render entryTree(entry, depth > 0 ? 16 : 0)} +{/snippet} + +{#snippet showMore({ + count, + onMore, + unit +}: { + count: number; + onMore: () => void; + unit: GroupedListUnit; +})} +
+ +
+{/snippet} + + (deleteOpen = false)} + open={deleteOpen} + repoWithTag={pendingDelete} +/> + +
+ + +
+ {@render sortHeader(ModelsTableSortKey.NAME, 'Model')} + + + {@render sortHeader(ModelsTableSortKey.CONTEXT, 'Context')} + + + + {@render sortHeader(ModelsTableSortKey.STATUS, 'Status')} + + + Actions +
+ +
+ {#each sections as group (group.key)} + {#if group.items.length > 0} + {#snippet groupIcon()} + {#if group.kind === ModelsTableGroupKind.DOWNLOADING} + + {:else if group.kind === ModelsTableGroupKind.FAVORITES} + + {:else if group.kind === ModelsTableGroupKind.LOADED} + + {:else if group.kind === ModelsTableGroupKind.HIDDEN} + + {:else if group.kind === ModelsTableGroupKind.LOCAL} + + {/if} + {/snippet} + + + entry.key} + more={showMore} + onCollapsedChange={(key, collapsed) => + modelsStore.setGroupCollapsed(group.key, key, collapsed)} + sectionWindow={MODEL_ROW_WINDOW} + stickyClass="bg-muted/30 backdrop-blur-lg" + stickyStyle={MODELS_TABLE_GROUP_STICKY_OFFSET} + weightOf={(entry) => entry.quants.length} + /> + + {/if} + {/each} + + {#if isEmpty} +

+ {hasFilters ? 'No models match these filters.' : 'No models found.'} +

+ {/if} +
+
diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerQuantRow.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerQuantRow.svelte new file mode 100644 index 000000000000..6d76251cdedd --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerQuantRow.svelte @@ -0,0 +1,71 @@ + + +
+ + + + + + + +
+ +
+
diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerRepoRow.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerRepoRow.svelte new file mode 100644 index 000000000000..8c26b201fa00 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerRepoRow.svelte @@ -0,0 +1,95 @@ + + +
+ + + + + + + + + + + {groupLabel} + + + + + + + + + + + {#if expanded} + + {:else} + + {/if} + +
diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerStatusCell.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerStatusCell.svelte new file mode 100644 index 000000000000..a56f50518e07 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerStatusCell.svelte @@ -0,0 +1,43 @@ + + +{#if download} + +{:else if isRouter} + +{:else} + + - + +{/if} diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerTableToolbar.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerTableToolbar.svelte new file mode 100644 index 000000000000..fef282fcc18b --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerTableToolbar.svelte @@ -0,0 +1,88 @@ + + + +
+ + + + + {#if hasFilters} + + {/if} + +
+ {@render toolbarEnd?.()} +
+
diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/row-actions.ts b/tools/ui/src/lib/components/app/models/ModelsManager/row-actions.ts new file mode 100644 index 000000000000..9931dd3f475f --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelsManager/row-actions.ts @@ -0,0 +1,61 @@ +import { Eye, EyeOff, Heart, HeartOff, Trash2 } from '@lucide/svelte'; +import { MODEL_DOWNLOAD_ICONS } from '$lib/constants'; +import { ModelRowDownloadState } from '$lib/enums'; +import { modelsStore } from '$lib/stores'; +import type { ModelOption } from '$lib/types/models'; + +/** Row actions follow the app's dropdown pattern: icon, label, separators, variants. */ +export function modelRowActions( + option: ModelOption, + favorite: boolean, + isHidden: boolean, + onDelete: (option: ModelOption) => void, + /** Download state, when the row stands for a tracked download. */ + download?: ModelRowDownloadState | null +) { + // a tracked download is paused, resumed or dropped: it is not loaded, hidden or + // drafted. Deleting it stops the download and removes what is on disk, which is + // what the same wording offers on a cached model + if (download) { + const isPaused = download === ModelRowDownloadState.PAUSED; + + return [ + { + icon: isPaused ? MODEL_DOWNLOAD_ICONS.resume : MODEL_DOWNLOAD_ICONS.pause, + label: isPaused ? 'Resume downloading' : 'Pause downloading', + onclick: () => + void (isPaused + ? modelsStore.status.downloadModel(option.model) + : modelsStore.status.pauseDownload(option.model)) + }, + { + icon: Trash2, + label: 'Delete from disk', + onclick: () => onDelete(option), + separator: true, + variant: 'destructive' as const + } + ]; + } + + return [ + { + icon: favorite ? HeartOff : Heart, + label: favorite ? 'Remove from favorites' : 'Add to favorites', + onclick: () => modelsStore.toggleFavorite(option.model) + }, + { + icon: Trash2, + label: 'Delete from disk', + onclick: () => onDelete(option), + separator: true, + variant: 'destructive' as const + }, + { + icon: isHidden ? Eye : EyeOff, + label: isHidden ? 'Unhide model' : 'Hide model', + onclick: () => modelsStore.toggleHidden(option.id), + separator: true + } + ]; +} diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/utils.ts b/tools/ui/src/lib/components/app/models/ModelsManager/utils.ts new file mode 100644 index 000000000000..3721ceca1091 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelsManager/utils.ts @@ -0,0 +1,202 @@ +import { LOCAL_BACKEND_ID, type ModalityKey } from '$lib/constants'; +import { + ModelCapability, + ModelGroupKind, + ModelsTableGroupKind, + ServerModelStatus +} from '$lib/enums'; +import { HuggingFaceService, ModelsService } from '$lib/services'; +import { modelsStore } from '$lib/stores'; +import type { ModelDownloadEntry, ModelDownloadProgress, ModelOption } from '$lib/types/models'; +import { detectThinkingSupport, detectToolUseSupport, repoOf } from '$lib/utils'; + +/** One repo of the table, with the rows it ships as. */ +export interface ModelQuantGroup { + base: ModelOption; + key: string; + kind: ModelGroupKind; + quants: ModelOption[]; +} + +/** True when any of the manager's filters narrows the table. */ +export function hasActiveFilters( + contextLimit: number, + modalities: ModalityKey[], + capabilities: ModelCapability[] +): boolean { + return contextLimit > 0 || modalities.length > 0 || capabilities.length > 0; +} + +/** + * Order a status sorts behind: loaded first, then a model being worked on + * (loading, sleeping), then the rest. + */ +export function statusRank(option: ModelOption): number { + const status = modelsStore.getModelStatus(option.model); + + if (modelsStore.isModelRunning(option.model)) return 2; + + if (status === ServerModelStatus.LOADING || status === ServerModelStatus.SLEEPING) return 1; + + return 0; +} + +/** Byte counts of a tracked download: live while it runs, frozen while paused. */ +export function downloadProgressFor(repoWithTag: string): ModelDownloadProgress | null { + return ( + modelsStore.status.getDownloadProgress(repoWithTag) ?? + modelsStore.status.getPausedDownloadProgress(repoWithTag) + ); +} + +/** + * One table row per tracked download, so the table lists them the way it lists + * any other model. A paused download is already in the router's listing, so its + * own option is reused; a fresh one stands for its tag alone. + */ +export function downloadGroups( + entries: ModelDownloadEntry[], + models: ModelOption[] +): ModelQuantGroup[] { + return entries.map((entry) => { + const option = models.find((candidate) => candidate.model === entry.repoWithTag) ?? { + backendId: LOCAL_BACKEND_ID, + capabilities: [], + id: entry.repoWithTag, + model: entry.repoWithTag, + name: entry.repoWithTag + }; + + return { + base: option, + key: `download::${entry.repoWithTag}`, + kind: ModelGroupKind.QUANTS, + quants: [option] + }; + }); +} + +/** One collapsible block of the manager's table. */ +export interface ModelsTableGroup { + /** One entry per repo, its quants hanging off it. */ + items: ModelQuantGroup[]; + key: string; + kind: ModelsTableGroupKind; + label: string; +} + +/** Context the model runs with: what a loaded model reports. */ +export function configuredContext(option: ModelOption): number | null { + return modelsStore.isModelRunning(option.model) + ? modelsStore.props.getModelContextSize(option.model) + : null; +} + +/** + * Capability a model reports: true or false once its listing or its chat template + * answers, null while the Hub record that carries the template is not read yet. + */ +export function modelCapability(option: ModelOption, capability: ModelCapability): boolean | null { + if (option.capabilities.includes(capability)) return true; + + const details = HuggingFaceService.cachedDetails(option.model); + + // only the chat template answers this, and it arrives with the Hub record + if (details === undefined) return null; + + const template = details?.gguf?.chat_template ?? ''; + + return capability === ModelCapability.TOOL_USE + ? detectToolUseSupport(template) + : detectThinkingSupport(template); +} + +/** + * Context a model reports: the provider listing first, then the cached Hub record. + * Null while neither of them has answered. + */ +export function modelContextLength(option: ModelOption): number | null { + return ( + option.contextLength ?? + HuggingFaceService.cachedDetails(option.model)?.gguf?.context_length ?? + null + ); +} + +/** Fold the quants of one repo into a single entry, so the table shows one row per model. */ +export function groupModelQuants(models: ModelOption[]): ModelQuantGroup[] { + const groups = new Map(); + + for (const option of models) { + const key = repoOf(option.model); + const group = groups.get(key); + + if (group) { + group.quants.push(option); + + continue; + } + + groups.set(key, { base: option, key, kind: ModelGroupKind.QUANTS, quants: [option] }); + } + + return Array.from(groups.values()).flatMap((group) => { + const kind = groupKind(group.quants); + const modelIds = group.quants.map((option) => option.model); + + // the very same id twice is not a quant set; keep those rows apart + if (modelIds.length > 1 && modelIds.every((model) => model === modelIds[0])) { + return group.quants.map((option) => ({ + ...group, + base: option, + key: `${group.key}::${option.id}`, + kind: ModelGroupKind.VARIANTS, + quants: [option] + })); + } + + return [{ ...group, kind }]; + }); +} + +/** What a group folds: quants or variants of one repo. */ +function groupKind(quants: ModelOption[]): ModelGroupKind { + const isQuant = quants.every( + (option) => (option.parsedId ?? ModelsService.parseModelId(option.model)).quantization + ); + + return isQuant ? ModelGroupKind.QUANTS : ModelGroupKind.VARIANTS; +} + +/** + * Split the repos of a section into the quants that are hidden and the ones that + * are not, so a partly hidden repo lists in both blocks instead of dragging its + * visible quants into the hidden one. + */ +export function splitHiddenQuants( + entries: ModelQuantGroup[], + isHidden: (option: ModelOption) => boolean +): { hidden: ModelQuantGroup[]; local: ModelQuantGroup[] } { + const hidden: ModelQuantGroup[] = []; + const local: ModelQuantGroup[] = []; + + for (const entry of entries) { + const hiddenQuants = entry.quants.filter((quant) => isHidden(quant)); + const localQuants = entry.quants.filter((quant) => !isHidden(quant)); + + if (hiddenQuants.length > 0) { + hidden.push({ + ...entry, + base: hiddenQuants[0], + key: `${entry.key}::hidden`, + quants: hiddenQuants + }); + } + + if (localQuants.length > 0) { + local.push({ ...entry, base: localQuants[0], quants: localQuants }); + } + } + + return { hidden, local }; +} diff --git a/tools/ui/src/lib/components/app/models/ModelsSection.svelte b/tools/ui/src/lib/components/app/models/ModelsSection.svelte new file mode 100644 index 000000000000..749dc5b0705e --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelsSection.svelte @@ -0,0 +1,97 @@ + + + modelsStore.setGroupOpen(persistKey, open) : null} + {revealChevronOnHover} + {triggerClass} + {triggerStyle} +> + {#snippet trigger()} + {#if onBack} + + {/if} + + {#if icon} + {@render icon()} + {/if} + + {label} + + {#if loading} + + {:else if error} + + {/if} + + {#if count !== undefined} + {count} + {/if} + {/snippet} + + {@render children()} + diff --git a/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelector.svelte b/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelector.svelte new file mode 100644 index 000000000000..33b3b2f86e6c --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelector.svelte @@ -0,0 +1,405 @@ + + +{#snippet selectorTriggerInner()} + + + + {#if selectedOption} + + {:else} + Select model + {/if} + + + {#if ms.updating || ms.isLoadingModel || triggerLoading} + + {:else} + + {/if} + + {#if triggerLoading} + + {/if} +{/snippet} + +{#snippet modelOption(item: ModelItem, hideOrgName: boolean)} + {@const { option } = item} + {@const isSelected = ms.activeId === option.id} + {@const isHighlighted = option.id === highlightedId} + {@const isFav = ms.isFavorite(option.model)} + + { + if (event.key === KeyboardKey.ENTER || event.key === KeyboardKey.SPACE) { + event.preventDefault(); + void handleModelKeyAction(option.id, event.altKey); + } + }} + onMouseEnter={() => (highlightedId = option.id)} + onSelect={ms.handleSelect} + {option} + /> +{/snippet} + +{#snippet pickerList(listClass: string)} +
+ {#if !ms.isCurrentModelInCache && displayModel} + + + {/if} + + {#if ms.isEmpty} +

{ms.emptyMessage}

+ {/if} + + +
+{/snippet} + +{#if ms.loading && ms.options.length === 0 && ms.isMultiModel} +
+ + + Loading models... +
+{:else if ms.options.length === 0 && ms.isMultiModel} + {#if displayModel} + + + + {:else} + No models yet. + {/if} +{:else if ms.isMultiModel} + {#if isMobile} + + + ms.setSearchTerm(v)} + onSearchKeyDown={handleSearchKeyDown} + searchTerm={ms.searchTerm} + > + {@render pickerList('px-2.5')} + + {:else} + (highlightedId = id)} + onModelKeyAction={(id, unload) => void handleModelKeyAction(id, unload)} + onSearchKeyDown={handleSearchKeyDown} + {showOrgName} + /> + {/if} +{:else if isMobile} + + + + ms.setSearchTerm(v)} + onSearchKeyDown={handleSearchKeyDown} + searchTerm={ms.searchTerm} + > + {@render pickerList('px-2.5')} + +{:else} + + + + + {#snippet child({ props })} + + {/snippet} + + +{/if} diff --git a/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDownloadItem.svelte b/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDownloadItem.svelte new file mode 100644 index 000000000000..6ef1433b970f --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDownloadItem.svelte @@ -0,0 +1,130 @@ + + + +
+ + + + onRequestCancel?.(entry.repoWithTag)} + /> + + {#if entry.progress && entry.progress.totalBytes > 0} + + {/if} +
diff --git a/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDrawer.svelte b/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDrawer.svelte new file mode 100644 index 000000000000..1a2e8d3b82a7 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDrawer.svelte @@ -0,0 +1,53 @@ + + + + + +
+ + {@render children()} + +
+ + diff --git a/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDropdown.svelte b/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDropdown.svelte new file mode 100644 index 000000000000..a7a54e8602a1 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDropdown.svelte @@ -0,0 +1,248 @@ + + +{#snippet selectorTriggerInner()} + + + + {#if selectedOption} + + {:else} + Select model + {/if} + + + {#if ms.updating || ms.isLoadingModel || triggerLoading} + + {:else} + + {/if} + + {#if triggerLoading} + + {/if} +{/snippet} + +{#snippet modelOption(item: ModelItem, hideOrgName: boolean)} + {@const { option } = item} + {@const isSelected = currentModel === option.model || ms.activeId === option.id} + {@const isHighlighted = option.id === highlightedId} + {@const isFav = ms.isFavorite(option.model)} + + { + if (event.key === KeyboardKey.ENTER || event.key === KeyboardKey.SPACE) { + event.preventDefault(); + onModelKeyAction?.(option.id, event.altKey); + } + }} + onMouseEnter={() => onHighlight?.(option.id)} + onSelect={ms.handleSelect} + {option} + /> +{/snippet} + + + + + + {#snippet child({ props })} + + {@render selectorTriggerInner()} + + {/snippet} + + + {#if selectedOption} + +

{triggerTooltipLabel(selectedOption)}

+ + {/if} + + + event.preventDefault()} + > + ms.setSearchTerm(v)} + {onSearchKeyDown} + placeholder="Search models..." + searchClass="bg-transparent" + searchValue={ms.searchTerm} + > + + {@render pickerContent('px-1.5')} + + {#snippet footer()} + + + + + Manage models + + + {/snippet} + + + + +{#snippet pickerContent(listClass: string)} +
+ {#if !ms.isCurrentModelInCache && currentModel} + + + {/if} + + {#if ms.isEmpty} +

{ms.emptyMessage}

+ {/if} + + +
+{/snippet} diff --git a/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorList.svelte b/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorList.svelte new file mode 100644 index 000000000000..6857e3018e64 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorList.svelte @@ -0,0 +1,235 @@ + + +{#snippet familyHeading({ group: family }: { group: ModelFamilyGroup })} +
+ + + + {family.label} + + + + {family.entries.length} model{family.entries.length === 1 ? '' : 's'} + +
+{/snippet} + +{#snippet listItem({ depth, entry }: { depth: number; entry: ModelItem })} +
{@render render(entry, !showOrgName)}
+{/snippet} + +{#snippet listRows(items: ModelItem[], prefix: string)} + {#if settingsStore.config[SETTINGS_KEYS.GROUP_MODELS_BY_FAMILY]} + row.option.model).map((family) => ({ + entries: family.entries, + group: family, + key: `${prefix}-${family.key}` + }))} + initialCollapsed={modelsStore.collapsedGroupsUnder(prefix)} + item={listItem} + keyOf={(row) => `${prefix}-${row.option.id}`} + onCollapsedChange={(key, collapsed) => modelsStore.setGroupCollapsed(prefix, key, collapsed)} + stickyStyle={SELECTOR_GROUP_STICKY_OFFSET} + /> + {:else} + `${prefix}-${row.option.id}`} /> + {/if} +{/snippet} + +{#snippet defaultOption(item: ModelItem, hideOrgName: boolean)} + {@const { option } = item} + {@const isSelected = currentModel === option.model || activeId === option.id} + {@const isFav = modelsStore.favoriteModelIds.has(option.model)} + + {}} + onMouseEnter={() => {}} + {onSelect} + {option} + /> +{/snippet} + +{#if loaded.length > 0} + + {#snippet icon()} + + {/snippet} + + {#each loaded as item (item.option.id)} + {@render render(item, !showOrgName)} + {/each} + +{/if} + +{#if favorites.length > 0} + + + {#snippet icon()} + + {/snippet} + + {#each favorites as item (`fav-${item.option.id}`)} + {@render render(item, !showOrgName)} + {/each} + +{/if} + +{#if downloadEntries.length > 0} +

Downloading

+ + {#each downloadEntries as entry (entry.repoWithTag)} + + {/each} +{/if} + +{#snippet localRows()} + {@render listRows( + localGroups.available.flatMap((group) => group.items), + 'local' + )} + + {#if hasMoreLocal} + + {/if} +{/snippet} + +{#if groups.available.length > 0} + + {#snippet icon()} + + {/snippet} + + {@render localRows()} + +{/if} + + (cancelOpen = false)} + open={cancelOpen} + repoWithTag={pendingCancel} +/> diff --git a/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorOption.svelte b/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorOption.svelte new file mode 100644 index 000000000000..2734573ca291 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorOption.svelte @@ -0,0 +1,105 @@ + + +
onSelect(option.id)} + onkeydown={onKeyDown} + onmouseenter={onMouseEnter} + role="option" + tabindex="0" + title={loadTitle} +> + + + + + {#if showCapabilities} + + {/if} + +
+ + + +
+ + {#if isLoading} + + {/if} +
diff --git a/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorTriggerIcon.svelte b/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorTriggerIcon.svelte new file mode 100644 index 000000000000..6c9621cddcb2 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorTriggerIcon.svelte @@ -0,0 +1,23 @@ + + +{#if option} + + {#snippet fallback()} + + {/snippet} + +{:else} + +{/if} diff --git a/tools/ui/src/lib/components/app/models/ModelsSelector/index.ts b/tools/ui/src/lib/components/app/models/ModelsSelector/index.ts new file mode 100644 index 000000000000..e234fbac33e0 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelsSelector/index.ts @@ -0,0 +1,10 @@ +/** Model selection UI: a desktop serves an anchored dropdown menu and a phone a + * bottom drawer, over their shared parts. */ + +export { default as ModelsSelector } from './ModelsSelector.svelte'; +export { default as ModelsSelectorDropdown } from './ModelsSelectorDropdown.svelte'; +export { default as ModelsSelectorDrawer } from './ModelsSelectorDrawer.svelte'; +export { default as ModelsSelectorList } from './ModelsSelectorList.svelte'; +export { default as ModelsSelectorDownloadItem } from './ModelsSelectorDownloadItem.svelte'; +export { default as ModelsSelectorOption } from './ModelsSelectorOption.svelte'; +export { default as ModelsSelectorTriggerIcon } from './ModelsSelectorTriggerIcon.svelte'; diff --git a/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte b/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte deleted file mode 100644 index 30dedcbae90a..000000000000 --- a/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte +++ /dev/null @@ -1,417 +0,0 @@ - - -
- {#if ms.loading && ms.options.length === 0 && ms.isRouter} -
- - - Loading models… -
- {:else if ms.options.length === 0 && ms.isRouter} - {#if currentModel} - - - - {:else} -

No models available.

- {/if} - {:else} - {@const selectedOption = ms.getDisplayOption()} - {@const triggerModel = selectedOption?.model} - {@const triggerStatus = triggerModel - ? modelsStore.routerModels.find((m) => m.id === triggerModel)?.status?.value - : undefined} - {@const triggerLoading = - !!triggerModel && - (triggerStatus === ServerModelStatus.LOADING || - modelsStore.status.isOperationInProgress(triggerModel))} - {@const triggerLoadPercent = triggerLoading - ? Math.round(modelLoadFraction(modelsStore.status.getLoadProgress(triggerModel)) * 100) - : 0} - - {#if ms.isRouter} - - - - - {#snippet child({ props })} - - - - - {#if selectedOption} - - {:else} - Select model - {/if} - - {#if reasoning.isReasoningActive} - - {/if} - - - {#if ms.updating || ms.isLoadingModel} - - {:else} - - {/if} - - {#if triggerLoading} - - {/if} - - {/snippet} - - - {#if selectedOption} - -

{selectedOption.model}

- - {/if} - - - event.preventDefault()} - > - - - - - {#if selectedOption} - - {:else} - No model - {/if} - - - - ms.setSearchTerm(v)} - onSearchKeyDown={handleSearchKeyDown} - placeholder="Search models..." - searchValue={ms.searchTerm} - > -
- {#if !ms.isCurrentModelInCache && currentModel} - - - {/if} - - {#if ms.filteredOptions.length === 0} -

No models found.

- {/if} - - {#snippet modelOption(item: ModelItem, hideOrgName: boolean)} - {@const { option } = item} - {@const isSelected = currentModel === option.model || ms.activeId === option.id} - {@const isHighlighted = option.id === highlightedId} - {@const isFav = ms.isFavorite(option.model)} - - { - if (event.key === KeyboardKey.ENTER || event.key === KeyboardKey.SPACE) { - event.preventDefault(); - void handleModelKeyAction(option.id, event.altKey); - } - }} - onMouseEnter={() => (highlightedId = option.id)} - onSelect={ms.handleSelect} - {option} - /> - {/snippet} - - -
- - - - - - - - {:else} - - - - {#snippet child({ props })} - - {/snippet} - - - {#if selectedOption} - -

{selectedOption.model}

- - {/if} - - {/if} - {/if} -
- -{#if ms.showModelDialog} - ms.setShowModelDialog(v)} - open={ms.showModelDialog} - /> -{/if} diff --git a/tools/ui/src/lib/components/app/models/ModelsSelectorList.svelte b/tools/ui/src/lib/components/app/models/ModelsSelectorList.svelte deleted file mode 100644 index e40e33d0c087..000000000000 --- a/tools/ui/src/lib/components/app/models/ModelsSelectorList.svelte +++ /dev/null @@ -1,76 +0,0 @@ - - -{#snippet defaultOption(item: ModelItem, hideOrgName: boolean)} - {@const { option } = item} - {@const isSelected = currentModel === option.model || activeId === option.id} - {@const isFav = modelsStore.favoriteModelIds.has(option.model)} - - {}} - onMouseEnter={() => {}} - {onSelect} - {option} - /> -{/snippet} - -{#if groups.loaded.length > 0} -

Loaded models

- - {#each groups.loaded as item (`loaded-${item.option.id}`)} - {@render render(item, false)} - {/each} -{/if} - -{#if groups.favorites.length > 0} -

Favorite models

- - {#each groups.favorites as item (`fav-${item.option.id}`)} - {@render render(item, true)} - {/each} -{/if} - -{#if groups.available.length > 0} -

Available models

- - {#each groups.available as group (group.orgName)} - {#if group.orgName} -

{group.orgName}

- {/if} - - {#each group.items as item (item.option.id)} - {@render render(item, true)} - {/each} - {/each} -{/if} diff --git a/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte b/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte deleted file mode 100644 index cee7d0c3b57c..000000000000 --- a/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte +++ /dev/null @@ -1,216 +0,0 @@ - - -
onSelect(option.id)} - onkeydown={onKeyDown} - onmouseenter={onMouseEnter} - role="option" - tabindex="0" - title={loadTitle} -> - - -
- - -
e.stopPropagation()} - > - {#if isFav} - modelsStore.toggleFavorite(option.model)} - tooltip="Remove from favorites" - /> - {:else} - modelsStore.toggleFavorite(option.model)} - tooltip="Add to favorites" - /> - {/if} - - - {#if isLoaded && onInfoClick} - onInfoClick(option.model)} - tooltip="Model information" - /> - {/if} -
- - {#if isLoading} -
- -
- {:else if isFailed} -
- - - -
- {:else if isSleeping} -
- - - -
- {:else if isLoaded} -
- - - -
- {:else} -
- - - -
- {/if} -
- - {#if isLoading} - - {/if} -
diff --git a/tools/ui/src/lib/components/app/models/ModelsSelectorSheet.svelte b/tools/ui/src/lib/components/app/models/ModelsSelectorSheet.svelte deleted file mode 100644 index c89ca186e3bb..000000000000 --- a/tools/ui/src/lib/components/app/models/ModelsSelectorSheet.svelte +++ /dev/null @@ -1,210 +0,0 @@ - - -
- {#if ms.loading && ms.options.length === 0 && ms.isRouter} -
- - Loading models… -
- {:else if ms.options.length === 0 && ms.isRouter} -

No models available.

- {:else} - {@const selectedOption = ms.getDisplayOption()} - {@const triggerModel = selectedOption?.model} - {@const triggerStatus = triggerModel - ? modelsStore.routerModels.find((m) => m.id === triggerModel)?.status?.value - : undefined} - {@const triggerLoading = - !!triggerModel && - (triggerStatus === ServerModelStatus.LOADING || - modelsStore.status.isOperationInProgress(triggerModel))} - {@const triggerLoadPercent = triggerLoading - ? Math.round(modelLoadFraction(modelsStore.status.getLoadProgress(triggerModel)) * 100) - : 0} - - {#if ms.isRouter} - - - - - - Select Model - - - Choose a model to use for the conversation - - - -
-
- ms.setSearchTerm(v)} - placeholder="Search models..." - value={ms.searchTerm} - /> -
- -
- {#if !ms.isCurrentModelInCache && currentModel} - - -
- {/if} - - {#if ms.filteredOptions.length === 0} -

No models found.

- {/if} - - -
-
- - - {:else} - - {/if} - {/if} -
- -{#if ms.showModelDialog} - ms.setShowModelDialog(v)} - open={ms.showModelDialog} - /> -{/if} diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverAvatar.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverAvatar.svelte deleted file mode 100644 index f78e7458644b..000000000000 --- a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverAvatar.svelte +++ /dev/null @@ -1,109 +0,0 @@ - - - - {#if orgAvatarFailed} - - {:else} -
- { - failedAvatarOrgs.add(org); - }} - src={HuggingFaceService.getAvatarUrl(org)} - /> -
- {/if} - - {#if quantOrg && quantOrg !== org} - - - {#if quantAvatarFailed} - - {:else} - { - failedAvatarOrgs.add(quantOrg ?? ''); - }} - src={HuggingFaceService.getAvatarUrl(quantOrg)} - /> - {/if} - - {/if} - diff --git a/tools/ui/src/lib/components/app/models/index.ts b/tools/ui/src/lib/components/app/models/index.ts index f6a9f1f9ba6d..dbfb5c60e248 100644 --- a/tools/ui/src/lib/components/app/models/index.ts +++ b/tools/ui/src/lib/components/app/models/index.ts @@ -10,70 +10,6 @@ * */ -/** - * **ModelsSelectorDropdown** - Model selection dropdown (desktop) - * - * Dropdown for selecting AI models with status indicators, - * search, and model information display. Adapts UI based on server mode. - * - * **Architecture:** - * - Uses DropdownMenuSearchable for model list - * - Integrates with modelsStore for model options and selection - * - Detects router vs single mode from serverStore - * - Opens DialogModelInformation for model details - * - * **Features:** - * - Searchable model list with keyboard navigation - * - Model status indicators (loading/ready/error/updating) - * - Model capabilities badges (vision, tools, etc.) - * - Current/active model highlighting - * - Model information dialog on info button click - * - Router mode: shows all available models with status - * - Single mode: shows current model name only - * - Loading/updating skeleton states - * - Global selection support for form integration - * - * @example - * ```svelte - * updateModel(id)} - * useGlobalSelection - * /> - * ``` - */ -export { default as ModelsSelectorDropdown } from './ModelsSelectorDropdown.svelte'; - -/** - * **ModelsSelectorList** - Grouped model options list - * - * Renders grouped model options (loaded, favorites, available) with section - * headers and org subgroups. Shared between ModelsSelectorDropdown and ModelsSelectorSheet - * to avoid template duplication. - * - * Accepts an optional `renderOption` snippet to customize how each option is - * rendered (e.g., to add keyboard navigation or highlighting). - */ -export { default as ModelsSelectorList } from './ModelsSelectorList.svelte'; - -/** - * **ModelsSelectorOption** - Single model option row - * - * Renders a single model option with selection state, favorite toggle, - * load/unload actions, status indicators, and an info button. - * Used inside ModelsSelectorList or directly in custom render snippets. - */ -export { default as ModelsSelectorOption } from './ModelsSelectorOption.svelte'; - -/** - * **ModelsSelectorSheet** - Mobile model selection sheet - * - * Bottom sheet variant of ModelsSelectorDropdown optimized for touch interaction - * on mobile devices. Same functionality as ModelsSelectorDropdown but uses Sheet UI - * instead of DropdownMenu. - */ -export { default as ModelsSelectorSheet } from './ModelsSelectorSheet.svelte'; - /** * **ModelBadge** - Model name display badge * * Compact badge showing current model name with package icon. @@ -108,13 +44,19 @@ export { default as ModelBadge } from './ModelBadge.svelte'; * aliases, and tags. Supports raw mode to show the unprocessed model name. * Respects the user's `showRawModelNames` setting. */ +export { default as ModelCapabilities } from './ModelCapabilities.svelte'; +export { default as ModelContext } from './ModelContext.svelte'; export { default as ModelId } from './ModelId.svelte'; +export { default as ModelAvatar } from './ModelAvatar.svelte'; +export { default as ModelDownloadProgressBar } from './ModelDownloadProgressBar.svelte'; +export { default as ModelLoadControl } from './ModelLoadControl.svelte'; +export { default as ModelOrgAvatar } from './ModelOrgAvatar.svelte'; +export { default as ModelRowActions } from './ModelRowActions.svelte'; +export { default as ModelsSection } from './ModelsSection.svelte'; -/** - * **ModelCapabilityIcons** - Capability and modality icon row - * - * The shared tool-use / reasoning / vision / video / audio icon cluster with - * tooltips, used by ModelId and the discover details header so the order and - * styling stay consistent across every model-id surface. - */ +/** Capability and modality icon row, shared by every model-id surface. */ export { default as ModelCapabilityIcons } from './ModelCapabilityIcons.svelte'; + +/** Draft sidecar badges, rendered as `+ [KIND] [QUANT]` per sidecar. */ +export { default as ModelDraftSidecars } from './ModelDraftSidecars.svelte'; +export * from './ModelsSelector'; diff --git a/tools/ui/src/lib/components/app/navigation/DropdownMenuActions.svelte b/tools/ui/src/lib/components/app/navigation/DropdownMenuActions.svelte index 20df04eb423d..161462a04be1 100644 --- a/tools/ui/src/lib/components/app/navigation/DropdownMenuActions.svelte +++ b/tools/ui/src/lib/components/app/navigation/DropdownMenuActions.svelte @@ -1,7 +1,10 @@ - - - - - {#snippet child({ props })} - e.stopPropagation()} - > - {@render iconComponent(triggerIcon, 'h-3 w-3')} + // a phone shows the same actions in a drawer, where a thumb reaches them and every + // row is a touch target of its own + let isMobile = $derived(deviceStore.isMobile); - {#if triggerTooltip} - {triggerTooltip} - {/if} - - {/snippet} - + function runAction(action: ActionItem, event: Event) { + open = false; + + // let the drawer close before an action that moves the focus, such as a confirm dialog + setTimeout(() => action.onclick(event), 0); + } + + +{#if isMobile} + + + + + + {triggerTooltip ?? 'Actions'} + + +
+ {#each actions as action, index (action.label)} + {#if action.separator && index > 0} + + {/if} + + + {/each} +
+ + +{:else} + + + + + {#snippet child({ props })} + e.stopPropagation()} + > + {@render iconComponent(triggerIcon, 'h-3 w-3')} + + {#if triggerTooltip} + {triggerTooltip} + {/if} + + {/snippet} + - - {#each actions as action, index (action.label)} - {#if action.separator && index > 0} - + {#if triggerTooltip} + +

{triggerTooltip}

+ {/if} + - -
- {@render iconComponent( - action.icon, - `h-4 w-4 ${action.variant === 'destructive' ? 'text-destructive' : ''}` - )} - {action.label} -
- - {#if action.shortcut} - + + {#each actions as action, index (action.label)} + {#if action.separator && index > 0} + {/if} - - {/each} - - + + +
+ {@render iconComponent( + action.icon, + `h-4 w-4 ${action.variant === 'destructive' ? 'text-destructive' : ''}` + )} + {action.label} +
+ + {#if action.shortcut} + + {/if} + + {/each} + + +{/if} {#snippet iconComponent(IconComponent: Component, className: string)} diff --git a/tools/ui/src/lib/components/app/navigation/DropdownMenuSearchable.svelte b/tools/ui/src/lib/components/app/navigation/DropdownMenuSearchable.svelte index f14db5f23c0f..79336cf29d96 100644 --- a/tools/ui/src/lib/components/app/navigation/DropdownMenuSearchable.svelte +++ b/tools/ui/src/lib/components/app/navigation/DropdownMenuSearchable.svelte @@ -1,50 +1,79 @@ -
+
-
- {@render children()} +
+ +
+ {@render children()} - {#if isEmpty} -
{emptyMessage}
- {/if} + {#if isEmpty} +
{emptyMessage}
+ {/if} +
{#if footer} - +
+ - {@render footer()} + {@render footer()} +
{/if} diff --git a/tools/ui/src/lib/components/app/navigation/SidebarNavigation/SidebarNavigation.svelte b/tools/ui/src/lib/components/app/navigation/SidebarNavigation/SidebarNavigation.svelte index 8bf12279e3c7..2ce4824cf930 100644 --- a/tools/ui/src/lib/components/app/navigation/SidebarNavigation/SidebarNavigation.svelte +++ b/tools/ui/src/lib/components/app/navigation/SidebarNavigation/SidebarNavigation.svelte @@ -5,6 +5,9 @@ import { ActionIcon, DialogConversationRename, + DialogManageModels, + DialogMcpServers, + DialogModelInformation, DialogSettingsChat, Logo, SidebarNavigationActions, @@ -93,6 +96,7 @@ let renameDialogOpen = $state(false); let settingsDialogOpen = $state(false); + let mcpDialogOpen = $state(false); let renameTargetConversationId = $state(null); let renameDraft = $state(''); let renameOriginalTitle = $state(''); @@ -306,6 +310,15 @@ pendingCollapse = null; } } + + /** A dialog owns the screen on a phone, so the expanded list steps aside for it. */ + function openDialog(open: () => void): void { + if (deviceStore.isMobile) { + scheduleMobileCollapse(); + } + + open(); + } @@ -365,9 +378,9 @@ > 768 ? uiStore.isSidebarExpanded : true} + onManageModelsClick={() => openDialog(() => uiStore.openModelsManager())} + onMcpClick={() => openDialog(() => (mcpDialogOpen = true))} onNewChat={() => { if (deviceStore.isMobile) { scheduleMobileCollapse(); @@ -402,7 +417,7 @@ isSearchModeActive = false; searchQuery = ''; }} - onSettingsClick={() => (settingsDialogOpen = true)} + onSettingsClick={() => openDialog(() => (settingsDialogOpen = true))} /> {#if uiStore.isSidebarExpanded || isOnMobile} @@ -452,6 +467,12 @@ + + + + + + \n' + f'\n\n

{html.escape(title)}

\n' + f'
{html.escape(meta)}
\n' + + "\n".join(parts) + "\n\n\n") + + +def write_transcript(messages: List[Dict[str, Any]], dest: Path, title: str, + meta: str = "") -> Path: + """Write `.json` (the raw message history) and `.html`.""" + dest.parent.mkdir(parents=True, exist_ok=True) + json_path = dest.with_name(dest.name + ".json") + with open(json_path, "w") as f: + json.dump(messages, f, indent=2, ensure_ascii=False) + with open(dest.with_name(dest.name + ".html"), "w") as f: + f.write(render_transcript_html(messages, title, meta)) + return json_path diff --git a/examples/llama-eval/eval_sandbox.py b/examples/llama-eval/eval_sandbox.py new file mode 100644 index 000000000000..b13bc9d7101b --- /dev/null +++ b/examples/llama-eval/eval_sandbox.py @@ -0,0 +1,488 @@ +#!/usr/bin/env python3 +"""Lightweight sandboxed execution for code-generation evals (HumanEval, ClassEval). + +Replaces the usual Docker-per-suite setup with a cached venv plus whatever +process isolation the host can give us without root: + + tier 1 bubblewrap read-only root, tmpfs /tmp and $HOME, no network + tier 2 unshare -rn no network (home still visible) + tier 3 rlimits only last resort, still capped on cpu/memory/procs/files + +Every tier also applies rlimits and runs the candidate program in a throwaway +working directory as its own process group, so a runaway solution is killed +along with anything it spawned. +""" + +import json +import os +import re +import resource +import shutil +import signal +import subprocess +import sys +import tempfile +import time +import venv +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Optional, Sequence + +DEFAULT_CACHE_ROOT = Path( + os.environ.get("LLAMA_EVAL_CACHE", Path.home() / ".cache" / "llama-eval") +) + +LIMIT_AS_BYTES = 4 * 1024 * 1024 * 1024 +LIMIT_CPU_SECONDS_SLACK = 5 +LIMIT_NPROC = 256 +LIMIT_FSIZE_BYTES = 64 * 1024 * 1024 + + +@dataclass +class ExecResult: + passed: bool + status: str + stdout: str = "" + stderr: str = "" + returncode: Optional[int] = None + duration: float = 0.0 + + def summary(self, max_chars: int = 400) -> str: + """Short human-readable reason, for grader logs.""" + if self.passed: + return "ok" + if self.status == "timeout": + return "timeout" + tail = (self.stderr or self.stdout).strip() + if not tail: + return f"{self.status} (rc={self.returncode})" + lines = [ln for ln in tail.splitlines() if ln.strip()] + reason = lines[-1] if lines else tail + return reason[:max_chars] + + +def resolve_python(spec: Optional[str]) -> Optional[str]: + """Turn '3.13' or a path into a usable interpreter, or None if unavailable. + + Suites can need an older interpreter than the one running this tool, because + a dependency has no wheel (or no working source build) for the newest + CPython. Accepts an explicit path, a bare version, or an executable name. + """ + if not spec: + return None + + candidate = Path(spec).expanduser() + if candidate.exists() and candidate.is_file(): + return str(candidate) + + uv = shutil.which("uv") + if uv and re.fullmatch(r"\d+\.\d+(\.\d+)?", spec): + try: + r = subprocess.run([uv, "python", "find", spec], + capture_output=True, text=True, timeout=120) + path = r.stdout.strip() + if r.returncode == 0 and path and Path(path).exists(): + return path + except Exception: + pass + + return shutil.which(spec) or shutil.which(f"python{spec}") + + +def _detect_isolation(probe_python: Optional[str] = None, + ro_binds: Sequence[Path] = ()) -> str: + """Pick the strongest isolation tier this host supports, once. + + The probe must run the *same* interpreter the suite will, with the same + binds. Probing a bare sys.executable silently downgrades the tier whenever + this tool is itself run from a venv under $HOME -- which is the usual case, + since it needs `datasets`. bwrap tmpfs-mounts the home directory, the + interpreter disappears, the probe fails, and the sandbox falls back to a + tier that does not hide $HOME at all. Nothing about the results looks wrong + when that happens, so it has to be got right here. + """ + py = probe_python or sys.executable + probe = "import sys; sys.exit(0)" + + if shutil.which("bwrap"): + try: + r = subprocess.run( + _bwrap_argv(Path.cwd(), [py, "-I", "-c", probe], ro_binds=ro_binds), + capture_output=True, timeout=20, + ) + if r.returncode == 0: + return "bwrap" + except Exception: + pass + + if shutil.which("unshare"): + for tier, argv in ( + ("unshare", ["unshare", "--user", "--map-current-user", "--net"]), + ("unshare-root", ["unshare", "-rn"]), + ): + try: + r = subprocess.run( + [*argv, py, "-I", "-c", probe], + capture_output=True, timeout=20, + ) + if r.returncode == 0: + return tier + except Exception: + continue + + return "rlimit" + + +UNSHARE_ARGV = { + "unshare": ["unshare", "--user", "--map-current-user", "--net"], + "unshare-root": ["unshare", "-rn"], +} + + +def _under(path: Path, parent: str) -> bool: + return bool(parent) and (str(path) == parent + or str(path).startswith(parent.rstrip("/") + os.sep)) + + +def _bwrap_argv(workdir: Path, inner: Sequence[str], + ro_binds: Sequence[Path] = ()) -> List[str]: + home = os.path.expanduser("~") + argv = [ + "bwrap", + "--ro-bind", "/", "/", + "--dev", "/dev", + "--proc", "/proc", + "--tmpfs", "/tmp", + ] + if home and home != "/": + argv += ["--tmpfs", home] + for path in ro_binds: + if _under(path, home): + argv += ["--ro-bind", str(path), str(path)] + argv += [ + "--bind", str(workdir), str(workdir), + "--chdir", str(workdir), + "--unshare-all", + "--die-with-parent", + "--new-session", + "--", + ] + return argv + list(inner) + + +def _nproc_ceiling() -> int: + """A process cap that bounds a fork bomb without tripping on existing load. + + RLIMIT_NPROC is enforced per real UID across the whole system, not per + process tree, so it has to sit above whatever the user is already running. + """ + try: + current = len(os.listdir("/proc")) + except OSError: + current = 512 + return max(current * 2, 1024) + + +def _apply_rlimits(timeout: float, with_nproc: bool, + as_bytes: Optional[int] = LIMIT_AS_BYTES): + def _preexec(): + os.setsid() + cpu = int(timeout) + LIMIT_CPU_SECONDS_SLACK + limits = [ + (resource.RLIMIT_CPU, (cpu, cpu)), + (resource.RLIMIT_FSIZE, (LIMIT_FSIZE_BYTES, LIMIT_FSIZE_BYTES)), + (resource.RLIMIT_CORE, (0, 0)), + ] + if as_bytes: + limits.append((resource.RLIMIT_AS, (as_bytes, as_bytes))) + if with_nproc: + ceiling = _nproc_ceiling() + limits.append((resource.RLIMIT_NPROC, (ceiling, ceiling))) + for res, lim in limits: + try: + resource.setrlimit(res, lim) + except (ValueError, OSError): + pass + return _preexec + + +class Sandbox: + """A cached venv plus an isolated runner for untrusted candidate programs.""" + + def __init__( + self, + name: str, + packages: Optional[Sequence[str]] = None, + cache_root: Optional[Path] = None, + base_python: Optional[str] = None, + isolation: Optional[str] = None, + provision: Optional[Sequence[str]] = None, + env: Optional[Dict[str, str]] = None, + address_space_limit: Optional[int] = LIMIT_AS_BYTES, + ): + self.name = name + self.packages = sorted(set(packages or [])) + self.provision = list(provision or []) + self.env = dict(env or {}) + # None drops the RLIMIT_AS cap entirely -- see _apply_rlimits + self.address_space_limit = address_space_limit + self.cache_root = Path(cache_root or DEFAULT_CACHE_ROOT) + self.base_python = base_python or sys.executable + self.venv_dir = self.cache_root / f"{name}-venv" + self._python: Optional[Path] = None + self._isolation = isolation + self._base_prefix: List[Path] = [] + + @property + def python(self) -> Path: + if self._python is None: + raise RuntimeError("Sandbox.ensure() must be called first") + return self._python + + @property + def isolation(self) -> str: + if self._isolation is None: + if self._python: + py = str(self._python) + roots = [self.venv_dir, *(self._base_prefix or [])] + else: + py = sys.executable + roots = self._detect_base_prefix(Path(py)) + self._isolation = _detect_isolation(py, roots) + return self._isolation + + def _stamp_path(self) -> Path: + return self.venv_dir / ".llama-eval-stamp.json" + + def _stamp_matches(self) -> bool: + try: + stamp = json.loads(self._stamp_path().read_text()) + except Exception: + return False + return ( + stamp.get("packages") == self.packages + and stamp.get("py") == self._base_version() + and stamp.get("provision") == self.provision + ) + + def _resolved_env(self) -> Dict[str, str]: + return {k: v.replace("{venv}", str(self.venv_dir)) for k, v in self.env.items()} + + def _base_version(self) -> str: + r = subprocess.run( + [self.base_python, "-c", + "import sys;print('.'.join(map(str,sys.version_info[:3])))"], + capture_output=True, text=True, + ) + return r.stdout.strip() + + def ensure(self, force: bool = False, quiet: bool = False) -> Path: + """Create/refresh the venv. Idempotent and cheap when already valid.""" + py = self.venv_dir / "bin" / "python" + if force and self.venv_dir.exists(): + shutil.rmtree(self.venv_dir) + + if py.exists() and self._stamp_matches(): + self._python = py + self._base_prefix = self._detect_base_prefix(py) + return py + + if not quiet: + pkgs = ", ".join(self.packages) if self.packages else "stdlib only" + print(f"[sandbox] provisioning venv {self.venv_dir} ({pkgs})") + + self.cache_root.mkdir(parents=True, exist_ok=True) + if self.venv_dir.exists(): + shutil.rmtree(self.venv_dir) + + uv = shutil.which("uv") + if uv: + subprocess.run( + [uv, "venv", "--python", self.base_python, str(self.venv_dir)], + check=True, capture_output=True, + ) + elif Path(self.base_python).resolve() == Path(sys.executable).resolve(): + venv.EnvBuilder(with_pip=bool(self.packages), clear=True).create(self.venv_dir) + else: + subprocess.run( + [self.base_python, "-m", "venv", str(self.venv_dir)], + check=True, capture_output=True, + ) + + if self.packages: + if uv: + cmd = [uv, "pip", "install", "--python", str(py), *self.packages] + else: + cmd = [str(py), "-m", "pip", "install", "--quiet", + "--disable-pip-version-check", *self.packages] + r = subprocess.run(cmd, capture_output=True, text=True) + if r.returncode != 0: + raise RuntimeError( + f"failed to install {len(self.packages)} package(s) into " + f"{self.venv_dir}:\n{(r.stderr or r.stdout)[-3000:]}" + ) + + for i, snippet in enumerate(self.provision, 1): + if not quiet: + print(f"[sandbox] provisioning step {i}/{len(self.provision)}") + prov_env = dict(os.environ) + prov_env["SANDBOX_VENV"] = str(self.venv_dir) + prov_env.update(self._resolved_env()) + r = subprocess.run([str(py), "-c", snippet], env=prov_env, + capture_output=True, text=True, timeout=1800) + if r.returncode != 0: + raise RuntimeError( + f"provisioning step {i} failed:\n{(r.stderr or r.stdout)[-3000:]}" + ) + + self._stamp_path().write_text(json.dumps({ + "packages": self.packages, + "py": self._base_version(), + "provision": self.provision, + })) + self._python = py + self._base_prefix = self._detect_base_prefix(py) + if not quiet: + print(f"[sandbox] ready ({self.isolation} isolation)") + if self.isolation == "unshare-root": + print("Warning: this host's unshare cannot keep the caller's uid, so " + "candidate code runs as root inside the namespace. Tests that " + "assert a permission denial will not see one. Install " + "bubblewrap (bwrap) for correct results.") + return py + + @staticmethod + def _detect_base_prefix(py: Path) -> List[Path]: + """Every directory that must stay visible for the venv to start. + + A venv is only symlinks into its base interpreter, so the whole install + tree has to survive the tmpfs that hides $HOME -- and binding just the + resolved tree is not enough. Version-managed interpreters (uv, pyenv) + are commonly reached through an unversioned alias symlink + (cpython-3.13-... -> cpython-3.13.14-...), and startup resolves through + the alias path, so both spellings need to be mounted. Walk the chain + and keep every install root it passes through. + """ + roots: List[Path] = [] + + def add(path: Path): + if path.exists() and path not in roots: + roots.append(path) + + cur, hops = py, 0 + while hops < 16: + hops += 1 + add(cur.parent.parent) + if cur.is_symlink(): + target = Path(os.readlink(cur)) + cur = target if target.is_absolute() else (cur.parent / target) + else: + break + + try: + r = subprocess.run([str(py), "-c", "import sys; print(sys.base_prefix)"], + capture_output=True, text=True, timeout=60) + if r.returncode == 0 and r.stdout.strip(): + add(Path(r.stdout.strip())) + except Exception: + pass + + return [p for p in roots if str(p) not in ("/", "/usr", "/usr/local")] + + def _isolate(self, argv: Sequence[str], workdir: Path, + ro_binds: Sequence[Path] = ()) -> List[str]: + """Wrap a command in the strongest isolation tier this host supports.""" + if self.isolation == "bwrap": + return _bwrap_argv( + workdir, argv, + ro_binds=[self.venv_dir, *(self._base_prefix or []), *ro_binds], + ) + if self.isolation in UNSHARE_ARGV: + return [*UNSHARE_ARGV[self.isolation], *argv] + return list(argv) + + def _base_env(self, workdir: Path) -> Dict[str, str]: + env = { + "PATH": "/usr/bin:/bin", + "HOME": str(workdir), + "TMPDIR": str(workdir), + "LC_ALL": "C.UTF-8", + "PYTHONHASHSEED": "0", + "PYTHONDONTWRITEBYTECODE": "1", + "OPENBLAS_NUM_THREADS": "1", + "OMP_NUM_THREADS": "1", + } + env.update(self._resolved_env()) + return env + + def run_argv(self, argv: Sequence[str], workdir: Path, + timeout: float = 15.0, env: Optional[Dict[str, str]] = None, + ro_binds: Sequence[Path] = ()) -> ExecResult: + """Run an arbitrary command in `workdir`, which stays writable. + + Unlike run(), the caller owns the directory and it survives the call. + That is what a multi-step agent needs: its edits have to persist across + many tool invocations before anything is graded. Taking an argv rather + than a source string is what lets a suite drive a linter or a test + runner for a language other than Python. + """ + run_env = self._base_env(workdir) + if env: + run_env.update(env) + + started = time.monotonic() + try: + proc = subprocess.Popen( + self._isolate(argv, workdir, ro_binds), + cwd=str(workdir), + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + errors="replace", + env=run_env, + preexec_fn=_apply_rlimits(timeout, + with_nproc=self.isolation == "rlimit", + as_bytes=self.address_space_limit), + ) + except Exception as e: + return ExecResult(False, "error", "", f"{type(e).__name__}: {e}") + + try: + out, err = proc.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + _kill_tree(proc) + out, err = proc.communicate() + return ExecResult(False, "timeout", out or "", err or "", + None, time.monotonic() - started) + + duration = time.monotonic() - started + if proc.returncode == 0: + return ExecResult(True, "ok", out, err, 0, duration) + return ExecResult(False, "failed", out, err, proc.returncode, duration) + + def run(self, code: str, timeout: float = 15.0, + env: Optional[Dict[str, str]] = None) -> ExecResult: + """Run `code` to completion in the sandbox. Exit 0 == passed.""" + workdir = Path(tempfile.mkdtemp(prefix=f"{self.name}-", dir="/tmp")) + try: + prog = workdir / "candidate.py" + prog.write_text(code) + return self.run_argv([str(self.python), "-I", "-B", str(prog)], + workdir, timeout=timeout, env=env) + except Exception as e: + return ExecResult(False, "error", "", f"{type(e).__name__}: {e}") + finally: + shutil.rmtree(workdir, ignore_errors=True) + + +def _kill_tree(proc: subprocess.Popen): + """Kill the candidate and everything it spawned.""" + for sig in (signal.SIGKILL,): + try: + os.killpg(os.getpgid(proc.pid), sig) + except (ProcessLookupError, PermissionError): + try: + proc.kill() + except ProcessLookupError: + pass diff --git a/examples/llama-eval/llama-eval.py b/examples/llama-eval/llama-eval.py index 61bdbddd8b13..3ed68e3b530d 100755 --- a/examples/llama-eval/llama-eval.py +++ b/examples/llama-eval/llama-eval.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import argparse +import ast import json import os import re @@ -8,23 +9,113 @@ import sys import threading import time +import uuid from abc import ABC, abstractmethod from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass, asdict, field from pathlib import Path from queue import Queue from typing import Dict, List, Optional, Any, Tuple +from urllib.parse import urlparse import requests from tqdm import tqdm import random from math import sqrt +from eval_sandbox import Sandbox, resolve_python + @dataclass class ServerConfig: url: str threads: int name: str = "" + api_key: Optional[str] = None + + def is_opencode(self) -> bool: + """Whether this endpoint is an opencode gateway rather than a llama-server.""" + return "opencode" in (urlparse(self.url).hostname or "") + + def headers(self, session_id: Optional[str] = None) -> Dict[str, str]: + headers = auth_headers(self.api_key) + if session_id and self.is_opencode(): + headers["x-opencode-session"] = session_id + return headers + + +def auth_headers(api_key: Optional[str]) -> Dict[str, str]: + """Standard request headers, carrying a bearer token when one is configured.""" + headers = {"Content-Type": "application/json"} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + return headers + + +SERVER_ERROR = "server_error" + +RETRYABLE_HTTP = {408, 409, 425, 429} + + +class ServerError(Exception): + """A request that still failed, or came back empty, after every retry.""" + + +def null_reply_reason(result: Any) -> Optional[str]: + """Why a chat completion carries no usable reply, or None when it does.""" + if not isinstance(result, dict): + return "reply is not a JSON object" + choices = result.get("choices") + if not choices: + return "reply has no choices" + msg = choices[0].get("message") + if not msg: + return "reply has no message" + if not (msg.get("content") or msg.get("tool_calls") or msg.get("reasoning_content")): + return "reply is empty" + return None + + +def post_with_retry(url: str, headers: Dict[str, str], data: Dict[str, Any], + retries: int, retry_delay: float, + timeout: Optional[float] = None) -> Dict[str, Any]: + """POST a chat completion, resending on errors and null replies. + + The wait before each resend doubles, starting at `retry_delay` seconds. + Raises ServerError once `retries` resends have all failed. + """ + reason = "" + attempts = 0 + for attempt in range(retries + 1): + if attempt: + wait = retry_delay * 2 ** (attempt - 1) + print(f" [{url}] {reason}; retry {attempt}/{retries} in {wait:.0f}s", + file=sys.stderr) + time.sleep(wait) + attempts += 1 + try: + response = requests.post(url, headers=headers, json=data, timeout=timeout) + except requests.RequestException as e: + reason = f"{type(e).__name__}: {e}" + continue + if response.status_code >= 400: + reason = f"HTTP {response.status_code}: {response.text[:300]}" + if response.status_code < 500 and response.status_code not in RETRYABLE_HTTP: + break + continue + try: + result = response.json() + except ValueError: + reason = f"invalid JSON: {response.text[:300]}" + continue + reason = null_reply_reason(result) or "" + if not reason: + return result + raise ServerError(f"{reason} (after {attempts} attempt{'s' if attempts > 1 else ''})") + + +def new_session_id(task_id: str) -> str: + """A fresh session id for one task, unique across tasks, runs and resumes.""" + return f"{task_id}-{uuid.uuid4().hex}" def wilson_interval(correct: int, total: int, z: float = 1.96) -> Tuple[float, float]: """Wilson score confidence interval for a proportion.""" @@ -106,13 +197,118 @@ def wilson_interval(correct: int, total: int, z: float = 1.96) -> Tuple[float, f B) {B} C) {C} D) {D} +""", + "humaneval": """Complete the following Python function. + +```python +{question} +``` + +Reply with the complete function in a single ```python code block, including the \ +signature and any imports it needs. Do not include tests, example usage, or explanation. +""", + "classeval": """Complete the following Python class. + +```python +{question} +``` + +Reply with the complete class in a single ```python code block: keep the class name \ +and every method signature exactly as given, implement every method body, and include \ +any imports the class needs. Do not include tests, example usage, or explanation. """, } +# Suites graded by running the generated code instead of matching its text. +CODE_SUITES = {"humaneval", "classeval", "agentic"} + +AGENTIC_SUITES = {"agentic"} + +# Import name -> pip name, for the cases where they differ. Anything not listed is assumed to install under the name it imports as. +PIP_NAMES = { + "bs4": "beautifulsoup4", + "PIL": "pillow", + "docx": "python-docx", + "fitz": "pymupdf", + "sklearn": "scikit-learn", + "yaml": "PyYAML", + "cv2": "opencv-python-headless", + "PyPDF2": "pypdf2", + "Levenshtein": "levenshtein", + "dateutil": "python-dateutil", + "attr": "attrs", + "OpenSSL": "pyOpenSSL", + "serial": "pyserial", + "Crypto": "pycryptodome", +} + +DATASET_SOURCE: Optional[str] = None + +# Extra construction options for the agentic suite, set from the CLI. +AGENTIC_OPTS: Dict[str, Any] = {} + class BaseDataset(ABC): questions: List[Dict] + is_agentic: bool = False + + def prepare(self, base_python: Optional[str] = None, force: bool = False) -> None: + """Provision whatever the suite needs before any task runs. + + Single-shot suites are provisioned from required_packages() instead, so + this is a no-op for them. + """ + + def run(self, index: int, chat) -> Dict[str, Any]: + """Drive one episode to completion and return its scored result.""" + raise NotImplementedError( + f"{type(self).__name__} does not drive its own conversation") + + def required_packages(self) -> List[str]: + """Third-party pip packages the suite's tests need.""" + return [] + + def provision_steps(self) -> List[str]: + """Python snippets run once at venv build time, with network access.""" + return [] + + def preferred_python(self) -> Optional[str]: + """Interpreter the suite's dependencies need, if not the current one.""" + return None + + def configure_python(self, python: str): + """Told which interpreter the sandbox will use, before it is built. + + Lets a suite adapt its package set to what that version can actually + install. + """ + + def default_exec_timeout(self) -> float: + """Per-task wall-clock limit that suits this suite's workload.""" + return 15.0 + + def sandbox_env(self) -> Dict[str, str]: + """Extra env for every run; '{venv}' expands to the venv path.""" + return {} + + def extract(self, response: str, question: Dict) -> str: + """Pull the candidate code out of a chat reply.""" + return extract_code(response) + + def build_program(self, question: Dict, completion: str, + seed: Optional[int] = None) -> str: + """Assemble the program whose exit status decides pass/fail.""" + raise NotImplementedError + + def interpret(self, result: Any, question: Dict) -> Tuple[bool, Dict[str, Any]]: + """Turn an ExecResult into (passed, detail-for-the-log).""" + return result.passed, { + "status": result.status, + "reason": result.summary(), + "duration": round(result.duration, 3), + } + @abstractmethod def get_question(self, index: int) -> Dict: pass @@ -175,6 +371,13 @@ def __init__( self.total_time: float = 0.0 self._lock = threading.Lock() + @property + def loaded_dataset(self) -> BaseDataset: + """The dataset, once loaded. Raises rather than handing back None.""" + if self.dataset is None: + raise RuntimeError("dataset not loaded; call load_dataset() first") + return self.dataset + def load_dataset(self, seed: int = 1234): if self.dataset_type == "aime": self.dataset = AimeDataset() @@ -186,6 +389,12 @@ def load_dataset(self, seed: int = 1234): self.dataset = Gsm8kDataset() elif self.dataset_type == "gpqa": self.dataset = GpqaDataset(variant="diamond", seed=seed) + elif self.dataset_type == "humaneval": + self.dataset = HumanEvalDataset(source=DATASET_SOURCE) + elif self.dataset_type == "classeval": + self.dataset = ClassEvalDataset(source=DATASET_SOURCE) + elif self.dataset_type == "agentic": + self.dataset = AgenticDataset(source=DATASET_SOURCE, **AGENTIC_OPTS) else: raise ValueError(f"Unknown dataset type: {self.dataset_type}") @@ -263,31 +472,55 @@ def add_result( self.correct = sum(1 for c in self.task_states.get("cases", {}).values() if c.get("correct", False)) + def _display_pair(self, task_state: TaskState) -> Tuple[str, str]: + """The (expected, answer) columns, kept narrow for code suites. + + There the gold value is a whole JSON test harness and the answer is a + whole function, so show the task id and the failure cause instead. + """ + if self.dataset_type not in CODE_SUITES: + return ( + task_state.expected, + task_state.answer if task_state.answer else "N/A", + ) + try: + expected = json.loads(task_state.expected)["task_id"] + except (ValueError, KeyError, TypeError): + expected = self.dataset_type + reason = (task_state.grader_log or {}).get("reason", "") + if task_state.correct: + reason = "ok" + return expected, (reason[:28] or "N/A") + def print_progress(self, task_state: TaskState, total_tasks: int, n_correct: int = 0): - display_answer = task_state.answer if task_state.answer else "N/A" + display_expected, display_answer = self._display_pair(task_state) display_tokens = str(task_state.tokens) if task_state.tokens is not None else "N/A" display_tps = f"{task_state.tps_gen:.1f}" if task_state.tps_gen is not None else "N/A" display_t_gen = f"{task_state.t_gen_ms/1000:.1f}" if task_state.t_gen_ms is not None else "N/A" display_server = task_state.server_name if task_state.server_name else "N/A" success_ratio = n_correct / self.processed if self.processed > 0 else 0.0 - first_line = task_state.question_text.split('\n')[0] + first_line = next( + (ln for ln in task_state.question_text.split('\n') if ln.strip()), "" + ) truncated_question = first_line[:43] if len(first_line) > 43: truncated_question += "..." else: truncated_question = truncated_question.ljust(43) + "..." - print(f"{self.processed:3}/{total_tasks:3} {task_state.task_id:<20} {self.dataset_type.upper()} {truncated_question:<40} {task_state.expected:<10} {display_answer:<10} {display_tokens:<6} {display_tps:<6} {display_t_gen:<8} {'✓' if task_state.correct else '✗'} [{n_correct:3}/{self.processed:3}, {success_ratio:.3f}] {display_server}") + print(f"{self.processed:3}/{total_tasks:3} {task_state.task_id:<20} {self.dataset_type.upper()} {truncated_question:<40} {display_expected:<16} {display_answer:<28} {display_tokens:<6} {display_tps:<6} {display_t_gen:<8} {'✓' if task_state.correct else ('⚠' if task_state.status == SERVER_ERROR else '✗')} [{n_correct:3}/{self.processed:3}, {success_ratio:.3f}] {display_server}") + + def n_server_errors(self) -> int: + cases = self.task_states.get("cases", {}) + return sum(1 for c in cases.values() if c.get("status") == SERVER_ERROR) + + def print_server_errors(self): + n = self.n_server_errors() + if n: + print(f"Server errors: {n} (not counted in accuracy; run with --resume to retry them)") def print_summary(self): - if self.total == 0: - print(f"\n{'='*60}") - print(f"Results: 0/0 correct (0.0%)") - print(f"{'='*60}") - else: - ci_lower, ci_upper = self.accuracy_ci() - print(f"\n{'='*60}") - print(f"Results: {self.correct}/{self.total} correct ({self.correct/self.total*100:.1f}%) [{ci_lower*100:.1f}%, {ci_upper*100:.1f}%]") - print(f"{'='*60}") + print() + self.print_existing_summary() def dump(self): with self._lock: @@ -347,7 +580,8 @@ def dump_html(self, tasks_to_save: List[Tuple[int, str]], all_cases: Dict[str, A completed = {tid: c for tid, c in cases.items() if c.get("status") == "ok"} n_correct = sum(1 for c in completed.values() if c.get("correct", False)) n_incorrect = len(completed) - n_correct - n_pending = len(tasks_to_save) - len(completed) + n_server_errors = sum(1 for c in cases.values() if c.get("status") == SERVER_ERROR) + n_pending = len(tasks_to_save) - len(completed) - n_server_errors accuracy = n_correct / len(completed) * 100 if completed else 0.0 ci_lower, ci_upper = wilson_interval(n_correct, len(completed)) if completed else (0.0, 1.0) @@ -374,6 +608,9 @@ def dump_html(self, tasks_to_save: List[Tuple[int, str]], all_cases: Dict[str, A elif status == "pending": status_class = "pending" status_text = "–" + elif status == SERVER_ERROR: + status_class = "server-error" + status_text = "⚠" else: status_class = "error" status_text = "!" @@ -392,6 +629,11 @@ def dump_html(self, tasks_to_save: List[Tuple[int, str]], all_cases: Dict[str, A escaped_reasoning = self._escape_html(reasoning_content) grader_log_str = self._escape_html(json.dumps(grader_log, indent=2)) escaped_server = self._escape_html(server_name) + transcript = grader_log.get("transcript") if isinstance(grader_log, dict) else None + transcript_link = ( + f'Transcript ' + f'{self._escape_html(transcript[:-5])}.html
' + if transcript and transcript.endswith(".json") else "") answer_class = status_class if status == "ok" else "" rows.append(f""" @@ -407,6 +649,7 @@ def dump_html(self, tasks_to_save: List[Tuple[int, str]], all_cases: Dict[str, A
+ {transcript_link} Prompt
{escaped_prompt}
Response
{escaped_response}
{f'Reasoning
{escaped_reasoning}
' if escaped_reasoning else ''} @@ -483,6 +726,7 @@ def _stat(v, fmt=".1f", avg_fmt=None): .incorrect {{ color: #cf222e; }} .pending {{ color: #888; }} .error {{ color: #9a6700; }} + .server-error {{ color: #8250df; }} .details-row {{ display: none; }} .details-row.open {{ display: table-row; }} .details-content {{ padding: 8px 16px; background: #f6f8fa; font-size: 12px; }} @@ -511,6 +755,7 @@ def _stat(v, fmt=".1f", avg_fmt=None):
Accuracy
{accuracy:.1f}% [{ci_lower*100:.1f}%, {ci_upper*100:.1f}%]
Correct
{n_correct} / {len(completed)}
Pending
{n_pending}
+
Server errors
{n_server_errors}
Time
{self.total_time:.1f}s
Sampling
{sampling_str}
@@ -680,6 +925,7 @@ def print_existing_summary(self): print(f"{'='*60}") print(f"Results: {correct}/{total} correct ({correct/total*100:.1f}%) [{ci_lower*100:.1f}%, {ci_upper*100:.1f}%]") print(f"{'='*60}") + self.print_server_errors() def accuracy_ci(self) -> Tuple[float, float]: """Compute Wilson score confidence interval from completed cases.""" @@ -974,6 +1220,526 @@ def get_prompt(self, question: Dict) -> str: D=question["shuffled_answers"][3] ) +FENCE_RE = re.compile(r"```[ \t]*([A-Za-z0-9_+-]*)[ \t]*\n(.*?)(?:```|\Z)", re.DOTALL) + +RESULT_MARKER = "__LLAMA_EVAL__" + +GENSIM_SHIM = { + "__init__.py": ( + '"""Minimal stand-in for the two gensim entry points ClassEval uses."""\n' + "from . import utils, matutils # noqa: F401\n" + ), + "utils.py": ( + "import re, html\n" + "RE_ENTITY = re.compile(r'&(#?)([xX]?)(\\w{1,8});')\n" + "def decode_htmlentities(text):\n" + " return RE_ENTITY.sub(lambda m: html.unescape(m.group(0)), text)\n" + ), + "matutils.py": ( + "import numpy as np\n" + "def unitvec(vec, norm='l2', return_norm=False):\n" + " v = np.asarray(vec, dtype=float)\n" + " n = float(np.sum(np.abs(v))) if norm == 'l1' " + "else float(np.sqrt(np.sum(v ** 2)))\n" + " if n == 0.0:\n" + " return (v, 1.0) if return_norm else v\n" + " out = v / n\n" + " return (out, n) if return_norm else out\n" + ), +} + +UNITTEST_PROLOGUE = ''' +import unittest as _ut_pre +_ut_pre.main = lambda *a, **k: None +''' + +UNITTEST_RUNNER = f''' + +if True: + import json as _json, os as _os, unittest as _ut + _loader = _ut.TestLoader() + _suite = _ut.TestSuite() + for _c in [v for v in list(globals().values()) + if isinstance(v, type) and issubclass(v, _ut.TestCase) + and v is not _ut.TestCase]: + _suite.addTests(_loader.loadTestsFromTestCase(_c)) + with open(_os.devnull, "w") as _null: + _res = _ut.TextTestRunner(stream=_null, verbosity=0).run(_suite) + _bad = {{t.id() for t, _ in _res.failures}} | {{t.id() for t, _ in _res.errors}} + _total = _res.testsRun + print("{RESULT_MARKER}" + _json.dumps({{ + "total": _total, + "passed": _total - len(_bad), + "failed": sorted(_bad), + "first_error": (_res.failures + _res.errors)[0][1].strip().splitlines()[-1] + if (_res.failures or _res.errors) else None, + }})) +''' + + +def parse_test_report(stdout: str) -> Optional[Dict[str, Any]]: + """Read the JSON line emitted by UNITTEST_RUNNER, if the program got that far.""" + for line in reversed(stdout.splitlines()): + if line.startswith(RESULT_MARKER): + try: + return json.loads(line[len(RESULT_MARKER):]) + except ValueError: + return None + return None + + +def extract_code(response: str, entry_point: Optional[str] = None, + class_name: Optional[str] = None) -> str: + """Pull the candidate program out of a chat reply. + + Prefers a fenced block that actually defines the thing being asked for, + since models like to emit a usage example or a test in a second block. + """ + if not response: + return "" + + if class_name: + wanted = rf"^\s*class\s+{re.escape(class_name)}\s*[:\(]" + elif entry_point: + wanted = rf"^\s*(?:async\s+)?def\s+{re.escape(entry_point)}\s*\(" + else: + wanted = None + + blocks = [(lang.lower(), body) for lang, body in FENCE_RE.findall(response)] + if blocks: + py = [b for lang, b in blocks if lang in ("python", "py", "python3", "")] + candidates = py or [b for _, b in blocks] + if wanted: + defining = [b for b in candidates if re.search(wanted, b, re.M)] + if defining: + return defining[-1].strip("\n") + return candidates[0].strip("\n") + + return response.strip("\n") + + +def _import_preamble(prompt: str) -> str: + """The import lines from the original stub. + + A chat model often returns just the function and drops the imports that the + stub had above it; replaying them keeps that from being scored as a failure. + """ + lines = [] + for line in prompt.splitlines(): + stripped = line.strip() + if stripped.startswith(("import ", "from ")) and not line.startswith((" ", "\t")): + lines.append(stripped) + return "\n".join(lines) + + +def build_program(question: Dict, completion: str, seed: Optional[int] = 0) -> str: + """Assemble the full program that decides pass/fail for one problem.""" + entry_point = question["entry_point"] + code = completion + + if not re.search(rf"^\s*(?:async\s+)?def\s+{re.escape(entry_point)}\s*\(", code, re.M): + body = completion if completion.startswith((" ", "\t")) else \ + "\n".join(" " + ln for ln in completion.splitlines()) + code = question["prompt"] + "\n" + body + + prologue = f"import random; random.seed({seed})" if seed is not None else "" + + return "\n\n".join(part for part in [ + _import_preamble(question["prompt"]), + code, + question["test"], + prologue, + f"check({entry_point})", + ] if part) + + +def pass_at_k(n: int, c: int, k: int) -> float: + """Unbiased pass@k estimator from the Codex paper (arXiv:2107.03374). + + n samples drawn, c of them correct. Computed as 1 - C(n-c,k)/C(n,k) via a + product so it stays stable for large n. + """ + if n - c < k: + return 1.0 + prod = 1.0 + for i in range(n - c + 1, n + 1): + prod *= 1.0 - k / i + return 1.0 - prod + + +class HumanEvalDataset(BaseDataset): + """OpenAI HumanEval, 164 hand-written Python problems. + + Every problem in this suite imports only from the standard library, so the + execution venv needs no pip installs at all -- see `required_packages`. + """ + + HF_REPO = "openai/openai_humaneval" + + def __init__(self, source: Optional[str] = None, split: str = "test"): + self.source = source or self.HF_REPO + self.split = split + self.questions: List[Dict] = [] + self._load_dataset() + + def _load_dataset(self): + print(f"Loading HumanEval dataset (source: {self.source})...") + + local = Path(self.source) + if local.exists(): + rows = [json.loads(ln) for ln in local.read_text().splitlines() if ln.strip()] + else: + from datasets import load_dataset + rows = [dict(r) for r in load_dataset(self.source, split=self.split)] + + required = {"task_id", "prompt", "entry_point", "test"} + for row in rows: + missing = required - row.keys() + if missing: + raise ValueError( + f"{self.source}: record {row.get('task_id', '?')} is missing {sorted(missing)}" + ) + row["dataset_type"] = "humaneval" + self.questions.append(row) + + print(f"HumanEval dataset loaded: {len(self.questions)} problems") + + def required_packages(self) -> List[str]: + """Third-party pip packages this suite needs. Derived, not hardcoded.""" + return sorted(third_party_imports( + "\n".join(q["prompt"] + "\n" + q["test"] for q in self.questions) + )) + + def extract(self, response: str, question: Dict) -> str: + return extract_code(response, question.get("entry_point")) + + def build_program(self, question: Dict, completion: str, + seed: Optional[int] = None) -> str: + return build_program(question, completion, seed=seed) + + def get_question(self, index: int) -> Dict: + return self.questions[index] + + def get_question_text(self, question: Dict) -> str: + return question["prompt"] + + def get_answer(self, question: Dict) -> str: + return json.dumps({ + "task_id": question["task_id"], + "entry_point": question["entry_point"], + "prompt": question["prompt"], + "test": question["test"], + }) + + def get_prompt(self, question: Dict) -> str: + return TEMPLATE_REGISTRY["humaneval"].format( + question=self.get_question_text(question).rstrip(), + ) + + +class ClassEvalDataset(BaseDataset): + """ClassEval: 100 class-level Python generation tasks. + + A task is a whole class judged by several independent unittest classes, so + it is scored twice -- class level (every test passes) and method level + (what fraction of tests passed), matching the upstream protocol. + """ + + HF_REPO = "ilintar/ClassEval" + + def __init__(self, source: Optional[str] = None): + self.source = source or self.HF_REPO + self.questions: List[Dict] = [] + self._gensim_shim = False + self._load_dataset() + + def _load_dataset(self): + print(f"Loading ClassEval dataset (source: {self.source})...") + + local = Path(self.source) + if local.exists(): + text = local.read_text() + rows = (json.loads(text) if local.suffix == ".json" + else [json.loads(ln) for ln in text.splitlines() if ln.strip()]) + else: + from datasets import load_dataset + rows = [dict(r) for r in load_dataset(self.source, split="test")] + + required = {"task_id", "skeleton", "test", "class_name"} + for row in rows: + missing = required - row.keys() + if missing: + raise ValueError( + f"{self.source}: record {row.get('task_id', '?')} is missing {sorted(missing)}" + ) + row["dataset_type"] = "classeval" + row["n_tests"] = _count_test_methods(row["test"]) + self.questions.append(row) + + print(f"ClassEval dataset loaded: {len(self.questions)} classes, " + f"{sum(q['n_tests'] for q in self.questions)} test methods") + + EXTRA_PACKAGES = [ + "lxml", + "scipy", + ] + + def configure_python(self, python: str): + """gensim stops at cp313, so on 3.14+ swap it for a small local shim.""" + try: + r = subprocess.run( + [python, "-c", "import sys; print('%d.%d' % sys.version_info[:2])"], + capture_output=True, text=True, timeout=60, + ) + major, minor = (int(x) for x in r.stdout.strip().split(".")) + except Exception: + return + self._gensim_shim = (major, minor) >= (3, 14) + + def required_packages(self) -> List[str]: + blob = "\n\n".join( + "\n\n".join(_as_text(q.get(f)) for f in + ("skeleton", "test", "solution_code", "import_statement")) + for q in self.questions + ) + found = {PIP_NAMES.get(m, m) for m in third_party_imports(blob)} + found |= set(self.EXTRA_PACKAGES) + if self._gensim_shim: + found.discard("gensim") + return sorted(found) + + NLTK_CORPORA = ("punkt_tab", "averaged_perceptron_tagger_eng", "wordnet") + + def provision_steps(self) -> List[str]: + steps = [] + + if any("nltk" in _as_text(q.get("skeleton")) + _as_text(q.get("test")) + for q in self.questions): + steps.append( + "import os, nltk\n" + "target = os.path.join(os.environ['SANDBOX_VENV'], 'nltk_data')\n" + "os.makedirs(target, exist_ok=True)\n" + f"for corpus in {self.NLTK_CORPORA!r}:\n" + " nltk.download(corpus, download_dir=target, quiet=True)\n" + ) + + if self._gensim_shim: + steps.append( + "import os, sysconfig\n" + "pkg = os.path.join(sysconfig.get_paths()['purelib'], 'gensim')\n" + "os.makedirs(pkg, exist_ok=True)\n" + f"files = {GENSIM_SHIM!r}\n" + "for name, body in files.items():\n" + " open(os.path.join(pkg, name), 'w').write(body)\n" + "import gensim.utils, gensim.matutils\n" + ) + + return steps + + def sandbox_env(self) -> Dict[str, str]: + return {"NLTK_DATA": "{venv}/nltk_data"} + + def preferred_python(self) -> Optional[str]: + return "3.13" + + def default_exec_timeout(self) -> float: + return 30.0 + + def extract(self, response: str, question: Dict) -> str: + return extract_code(response, class_name=question.get("class_name")) + + def build_program(self, question: Dict, completion: str, + seed: Optional[int] = None) -> str: + prologue = f"import random; random.seed({seed})" if seed is not None else "" + return "\n\n".join(part for part in [ + _as_text(question.get("import_statement")), + UNITTEST_PROLOGUE, + completion, + question["test"], + prologue, + UNITTEST_RUNNER, + ] if part) + + def interpret(self, result: Any, question: Dict) -> Tuple[bool, Dict[str, Any]]: + expected = question.get("n_tests", 0) + + report = parse_test_report(result.stdout) + if report is None: + return False, { + "status": result.status if result.status != "ok" else "no-report", + "reason": result.summary(), + "tests_total": expected, + "tests_passed": 0, + "duration": round(result.duration, 3), + } + + total, passed = report.get("total", 0) or expected, report.get("passed", 0) + ok = total > 0 and passed == total + return ok, { + "status": "ok" if ok else "failed", + "reason": "ok" if ok else (report.get("first_error") + or f"{passed}/{total} tests passed")[:200], + "tests_total": total, + "tests_passed": passed, + "duration": round(result.duration, 3), + } + + def get_question(self, index: int) -> Dict: + return self.questions[index] + + def get_question_text(self, question: Dict) -> str: + return question["skeleton"] + + def get_answer(self, question: Dict) -> str: + return json.dumps({ + "task_id": question["task_id"], + "class_name": question["class_name"], + "import_statement": _as_text(question.get("import_statement")), + "test": question["test"], + "n_tests": question["n_tests"], + }) + + def get_prompt(self, question: Dict) -> str: + return TEMPLATE_REGISTRY["classeval"].format( + question=self.get_question_text(question).rstrip(), + ) + + +class AgenticDataset(BaseDataset): + """Multi-turn tool-driven repair tasks over a small real repository. + + The other code suites answer one prompt. Here the model is handed a tool + set and drives its own conversation, so the Processor delegates the whole + episode rather than making a single request. + + Everything language-specific is imported lazily and provisioned per + language, so selecting the Python tasks never installs a node toolchain -- + and running any other suite installs none of it. + """ + + HF_REPO = "ilintar/SACB" + is_agentic = True + ae: Any + + def __init__(self, source: Optional[str] = None, lang: Optional[str] = None, + max_turns: int = 50, context_limit: Optional[int] = None, + max_tokens: Optional[int] = None): + self.source = source or self.HF_REPO + self.lang_filter = lang + self.max_turns = max_turns + self.context_limit = context_limit + self.max_tokens = max_tokens + self._sandboxes: Dict[str, Any] = {} + self._load_dataset() + + def _load_dataset(self): + import agentic_eval as ae + self.ae = ae + + print(f"Loading agentic dataset (source: {self.source})...") + path = Path(self.source) + if path.exists(): + rows = [json.loads(line) for line in path.read_text().splitlines() + if line.strip()] + else: + from datasets import load_dataset + rows = [dict(r) for r in load_dataset(self.source, split="test")] + + if self.lang_filter: + rows = [r for r in rows if r.get("lang") == self.lang_filter] + if not rows: + raise ValueError( + f"no agentic tasks in {self.source}" + + (f" for language {self.lang_filter!r}" if self.lang_filter else "")) + + self.tasks = [ae.AgenticTask.from_record(r) for r in rows] + self.languages = sorted({t.lang for t in self.tasks}) + print(f"Agentic dataset loaded: {len(self.tasks)} tasks " + f"({', '.join(self.languages)})") + + def __len__(self) -> int: + return len(self.tasks) + + def get_question(self, index: int) -> Dict: + return {"index": index} + + def get_question_text(self, question: Dict) -> str: + task = self.tasks[question["index"]] + first = task.instruction.strip().splitlines()[0] + return f"[{task.task_id}] {first}" + + def get_answer(self, question: Dict) -> str: + task = self.tasks[question["index"]] + return json.dumps({"task_id": task.task_id, "index": question["index"]}) + + def get_prompt(self, question: Dict) -> str: + return self.ae.user_prompt(self.tasks[question["index"]]) + + def default_exec_timeout(self) -> float: + return 90.0 + + def required_packages(self) -> List[str]: + return [] + + def prepare(self, base_python: Optional[str] = None, force: bool = False): + """Provision one sandbox per language present in the selected tasks.""" + for lang in self.languages: + sandbox = self.ae.make_sandbox(lang, Sandbox, base_python=base_python) + sandbox.ensure(force=force) + self._sandboxes[lang] = sandbox + return self._sandboxes + + def run(self, index: int, chat) -> Dict[str, Any]: + task = self.tasks[index] + return self.ae.run_task( + task, chat, self._sandboxes[task.lang], + max_turns=self.max_turns, context_limit=self.context_limit, + test_timeout=self.default_exec_timeout()) + + +def _as_text(value: Any) -> str: + """Some dataset fields arrive as a list of lines rather than a blob.""" + if not value: + return "" + if isinstance(value, (list, tuple)): + return "\n".join(str(v) for v in value) + return str(value) + + +def _count_test_methods(test_code: str) -> int: + """How many `def test_*` a task's test suite defines.""" + try: + tree = ast.parse(test_code) + except SyntaxError: + return len(re.findall(r"^\s*def\s+test\w*\s*\(", test_code, re.M)) + return sum( + 1 + for node in ast.walk(tree) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name.startswith("test") + ) + + +def third_party_imports(source: str) -> set: + """Top-level modules imported by `source` that are not in the stdlib.""" + import ast + + mods = set() + for chunk in source.split("\n\n"): + try: + tree = ast.parse(chunk) + except SyntaxError: + continue + for node in ast.walk(tree): + if isinstance(node, ast.Import): + mods.update(a.name.split(".")[0] for a in node.names) + elif isinstance(node, ast.ImportFrom) and node.module and node.level == 0: + mods.add(node.module.split(".")[0]) + + stdlib = set(sys.stdlib_module_names) + return {m for m in mods if m and m not in stdlib} + + class Grader: def __init__( self, @@ -981,14 +1747,29 @@ def __init__( grader_script: Optional[str] = None, grader_model_name: Optional[str] = None, grader_server_url: str = "", - dataset_type: str = "aime" + grader_api_key: Optional[str] = None, + dataset_type: str = "aime", + sandbox: Optional[Sandbox] = None, + exec_timeout: float = 15.0, + exec_seed: Optional[int] = 0, + dataset: Optional[BaseDataset] = None, ): self.grader_type = grader_type self.grader_script = grader_script self.grader_model_name = grader_model_name self.grader_server_url = grader_server_url + self.grader_api_key = grader_api_key self.dataset_type = dataset_type + self.sandbox = sandbox + self.exec_timeout = exec_timeout + self.exec_seed = exec_seed + self.dataset = dataset self.pattern = self._get_pattern() + self._local = threading.local() + + @property + def last_exec(self) -> Dict[str, Any]: + return getattr(self._local, "exec_info", {}) def _get_pattern(self) -> Optional[str]: if self.grader_type == "regex": @@ -1077,7 +1858,7 @@ def _grade_llm(self, gold: str, pred: str, problem: str) -> Tuple[bool, Optional Please provide only the extracted answer, nothing else. If there is no clear answer that can be extracted from the response, reply with 'no answer'.""" url = f"{self.grader_server_url}/v1/chat/completions" - headers = {"Content-Type": "application/json"} + headers = auth_headers(self.grader_api_key) data = { "model": self.grader_model_name, "messages": [ @@ -1102,6 +1883,28 @@ def _truncate_response(self, response: str, max_lines: int = 6) -> str: lines = response.split('\n') return '\n'.join(lines[-max_lines:]) if len(lines) > max_lines else response + def _grade_exec(self, gold: str, pred: str) -> Tuple[bool, Optional[str]]: + """Grade by running the generated code against the suite's own tests. + + `gold` is the JSON harness produced by the dataset's get_answer(); the + dataset owns how the program is assembled and how its result is read. + """ + if self.sandbox is None or self.dataset is None: + raise ValueError("exec grader requires a sandbox and a dataset") + + question = json.loads(gold) + completion = self.dataset.extract(pred, question) + if not completion.strip(): + self._local.exec_info = {"status": "empty", "reason": "no code in reply"} + return False, None + + program = self.dataset.build_program(question, completion, seed=self.exec_seed) + result = self.sandbox.run(program, timeout=self.exec_timeout) + passed, detail = self.dataset.interpret(result, question) + + self._local.exec_info = detail + return passed, completion + def grade(self, gold: str, pred: str, problem: str = "") -> Tuple[bool, Optional[str]]: """Grade the response""" if self.grader_type == "regex": @@ -1110,6 +1913,8 @@ def grade(self, gold: str, pred: str, problem: str = "") -> Tuple[bool, Optional return self._grade_cli(gold, pred) elif self.grader_type == "llm": return self._grade_llm(gold, pred, problem) + elif self.grader_type == "exec": + return self._grade_exec(gold, pred) else: raise ValueError(f"Unknown grader type: {self.grader_type}") @@ -1119,18 +1924,24 @@ def __init__( server_configs: List[ServerConfig], grader: Grader, model_name: Optional[str] = None, - n_predict: int = -1 + n_predict: int = -1, + retries: int = 5, + retry_delay: float = 10.0, + transcript_dir: Optional[Path] = None, ): self.server_configs = server_configs self.grader = grader self.model_name = model_name self.n_predict = n_predict + self.retries = retries + self.retry_delay = retry_delay + self.transcript_dir = transcript_dir @staticmethod def _check_server(server_config: ServerConfig) -> List[str]: url = f"{server_config.url}/v1/models" try: - response = requests.get(url) + response = requests.get(url, headers=server_config.headers()) response.raise_for_status() models = [m["id"] for m in response.json().get("data", [])] return models @@ -1139,10 +1950,11 @@ def _check_server(server_config: ServerConfig) -> List[str]: sys.exit(1) def _make_request( - self, server_config: ServerConfig, eval_state: EvalState, prompt: str + self, server_config: ServerConfig, eval_state: EvalState, prompt: str, + session_id: Optional[str] = None ) -> Tuple[Dict[str, Any], int, Optional[float], Optional[float], str]: url = f"{server_config.url}/v1/chat/completions" - headers = {"Content-Type": "application/json"} + headers = server_config.headers(session_id) data = { "model": self.model_name if self.model_name else "llama", "messages": [{"role": "user", "content": prompt}], @@ -1156,10 +1968,10 @@ def _make_request( data["top_p"] = eval_state.sampling_config["top_p"] if eval_state.sampling_config.get("min_p") is not None: data["min_p"] = eval_state.sampling_config["min_p"] + if eval_state.sampling_config.get("reasoning_effort") is not None: + data["reasoning_effort"] = eval_state.sampling_config["reasoning_effort"] - response = requests.post(url, headers=headers, json=data) - response.raise_for_status() - result = response.json() + result = post_with_retry(url, headers, data, self.retries, self.retry_delay) tokens = result.get("usage", {}).get("completion_tokens", 0) timings = result.get("timings", {}) tps_gen = timings.get("predicted_per_second") if timings else None @@ -1187,8 +1999,14 @@ def _process_single_case( problem_idx=problem_idx, ) + session_id = new_session_id(task_id) + + if eval_state.loaded_dataset.is_agentic: + return self._process_agentic_case( + server_config, eval_state, i, task_id, task_state, session_id) + try: - response, tokens, tps_gen, t_gen_ms, finish_reason = self._make_request(server_config, eval_state, prompt) + response, tokens, tps_gen, t_gen_ms, finish_reason = self._make_request(server_config, eval_state, prompt, session_id) result = response["choices"][0]["message"]["content"] reasoning_content = response["choices"][0].get("message", {}).get("reasoning_content") task_state.response = result @@ -1208,15 +2026,20 @@ def _process_single_case( eval_state.dump() return task_state - result_truncated = self.grader._truncate_response(result, max_lines=10) - is_correct, answer = self.grader.grade(expected, result_truncated, prompt) + if self.grader.grader_type == "exec": + graded_text = result + else: + graded_text = self.grader._truncate_response(result, max_lines=10) + is_correct, answer = self.grader.grade(expected, graded_text, prompt) grader_log = { - "pred": result_truncated, + "pred": graded_text, "grader_type": self.grader.grader_type } if self.grader.grader_type == "regex" and self.grader.pattern: grader_log["pattern"] = self.grader.pattern + if self.grader.grader_type == "exec": + grader_log.update(self.grader.last_exec) task_state.correct = is_correct task_state.answer = answer @@ -1232,11 +2055,136 @@ def _process_single_case( eval_state.dump() + except ServerError as e: + task_state.status = SERVER_ERROR + task_state.grader_log = {"error": str(e)} + eval_state.add_result( + task_id, prompt, expected, None, None, task_state.grader_log, + False, SERVER_ERROR, None, None, None, None, server_config.name, + chunk_idx, problem_idx, + ) + eval_state.dump() except Exception as e: task_state.status = f"error: {str(e)}" return task_state + def _agentic_chat(self, server_config: ServerConfig, eval_state: EvalState, + session_id: Optional[str] = None): + """A chat callable for agentic_eval: messages + tools in, response out.""" + url = f"{server_config.url}/v1/chat/completions" + + def chat(messages, tools): + data = { + "model": self.model_name if self.model_name else "llama", + "messages": messages, + "tools": tools, + "tool_choice": "auto", + } + per_turn = getattr(eval_state.dataset, "max_tokens", None) + if self.n_predict and self.n_predict > 0: + data["max_tokens"] = self.n_predict + elif per_turn: + data["max_tokens"] = per_turn + for key in ("temperature", "top_k", "top_p", "min_p", "reasoning_effort"): + value = eval_state.sampling_config.get(key) + if value is not None: + data[key] = value + return post_with_retry(url, server_config.headers(session_id), data, + self.retries, self.retry_delay, timeout=1800) + + return chat + + def _process_agentic_case( + self, server_config: ServerConfig, eval_state: EvalState, i: int, + task_id: str, task_state: TaskState, session_id: Optional[str] = None + ) -> TaskState: + """Run one whole tool-driven episode and grade the tree it leaves behind.""" + try: + result = eval_state.loaded_dataset.run( + i, self._agentic_chat(server_config, eval_state, session_id)) + except Exception as e: + task_state.status = f"error: {type(e).__name__}: {e}" + eval_state.add_result( + task_id, task_state.prompt, task_state.expected, + task_state.status, None, + {"error": f"{type(e).__name__}: {e}", "resolved": False}, + False, task_state.status, 0, None, None, None, + server_config.name, task_state.chunk_idx, i, + ) + eval_state.dump() + return task_state + + episode = result.get("episode", {}) + messages = result.pop("messages", None) + if messages is not None and self.transcript_dir is not None: + self._save_transcript(eval_state, task_id, messages, result) + + if episode.get("stop_reason") in ("request_failed", "no_choices"): + task_state.status = SERVER_ERROR + task_state.grader_log = result + task_state.response = f"server error on turn {episode.get('turns')}: {episode.get('error')}" + eval_state.add_result( + task_id, task_state.prompt, task_state.expected, task_state.response, + None, result, False, SERVER_ERROR, + episode.get("completion_tokens", 0), None, None, None, + server_config.name, task_state.chunk_idx, i, + ) + eval_state.dump() + return task_state + + task_state.tokens = episode.get("completion_tokens", 0) + task_state.correct = bool(result["resolved"]) + task_state.answer = ", ".join(result.get("changed_files", [])) or None + task_state.grader_log = result + task_state.status = "ok" + task_state.response = ( + f"stop={episode.get('stop_reason')} turns={episode.get('turns')} " + f"calls={episode.get('tool_calls')} edits={episode.get('edits')} " + f"errors={episode.get('tool_errors')} nudges={episode.get('nudges')} truncated={episode.get('truncated_turns')} " + f"peak_ctx={episode.get('peak_context')}\n" + f"changed: {task_state.answer or '(nothing)'}" + ) + + eval_state.add_result( + task_id, task_state.prompt, task_state.expected, task_state.response, + task_state.answer, result, task_state.correct, "ok", + task_state.tokens, None, None, None, server_config.name, + task_state.chunk_idx, i, + ) + eval_state.dump() + return task_state + + def _save_transcript(self, eval_state: EvalState, task_id: str, + messages: List[Dict[str, Any]], result: Dict[str, Any]): + """Write the episode's messages as .json plus a readable .html.""" + assert self.transcript_dir is not None + ep = result.get("episode", {}) + verdict = "RESOLVED" if result.get("resolved") else "not resolved" + meta = ( + f"{verdict} f2p {result.get('f2p_passed')}/{result.get('f2p_total')} " + f"p2p {result.get('p2p_passed')}/{result.get('p2p_total')}\n" + f"stop={ep.get('stop_reason')} turns={ep.get('turns')} " + f"calls={ep.get('tool_calls')} edits={ep.get('edits')} " + f"errors={ep.get('tool_errors')} nudges={ep.get('nudges')} truncated={ep.get('truncated_turns')} " + f"prompt_tokens={ep.get('prompt_tokens')} " + f"completion_tokens={ep.get('completion_tokens')} " + f"peak_ctx={ep.get('peak_context')}\n" + f"changed: {', '.join(result.get('changed_files') or []) or '(nothing)'}" + + (f"\nerror: {ep.get('error')}" if ep.get("error") else "") + ) + title = f"{task_id} ({result.get('task_id')})" + dataset = eval_state.loaded_dataset + assert isinstance(dataset, AgenticDataset) + try: + path = dataset.ae.write_transcript( + messages, self.transcript_dir / task_id, title, meta) + except OSError as e: + print(f"Warning: could not write transcript for {task_id}: {e}") + return + base = Path(eval_state.output_file).resolve().parent + result["transcript"] = os.path.relpath(path.resolve(), base) + @staticmethod def _worker( server_config: ServerConfig, @@ -1275,7 +2223,7 @@ def evaluate(self, eval_state: EvalState, verbose: bool = False, resume: bool = print(f" {i+1}. {sc.name} — {sc.url} ({sc.threads} threads) [{models_str}]") print(f"Model: {self.model_name}") print(f"Grader: {self.grader.grader_type}") - print(f"Sampling: temp={eval_state.sampling_config.get('temperature', 'skip')}, top-k={eval_state.sampling_config.get('top_k', 'skip')}, top-p={eval_state.sampling_config.get('top_p', 'skip')}, min-p={eval_state.sampling_config.get('min_p', 'skip')}") + print(f"Sampling: temp={eval_state.sampling_config.get('temperature', 'skip')}, top-k={eval_state.sampling_config.get('top_k', 'skip')}, top-p={eval_state.sampling_config.get('top_p', 'skip')}, min-p={eval_state.sampling_config.get('min_p', 'skip')}, reasoning-effort={eval_state.sampling_config.get('reasoning_effort', 'skip')}") print() # Shared task queue: all workers compete for tasks @@ -1342,6 +2290,79 @@ def evaluate(self, eval_state: EvalState, verbose: bool = False, resume: bool = eval_state.print_summary() eval_state.dump() +def preflight_sandbox(sandbox: Sandbox): + """Catch host policies that would fail problems for non-model reasons. + + A FIPS-enforcing OpenSSL makes hashlib.md5() raise, which would sink + HumanEval/162 no matter what the model generated. + """ + probe = sandbox.run("import hashlib; hashlib.md5(b'x').hexdigest()", timeout=15) + if not probe.passed: + print(f"Warning: md5 is unavailable in the execution venv " + f"({probe.summary(120)}). Problems that hash with md5 will fail " + f"regardless of the model -- this is a host policy, not a model result.") + + +def print_pass_at_k(eval_state: "EvalState"): + """Report pass@k plus a breakdown of how the failures failed. + + Each chunk is one full pass over the suite, so running --n_cases at a + multiple of the dataset size gives n samples per problem for free. + """ + cases = eval_state.task_states.get("cases", {}) + if not cases: + return + + by_problem: Dict[int, List[bool]] = {} + reasons: Dict[str, int] = {} + for case in cases.values(): + by_problem.setdefault(case.get("problem_idx", 0), []).append( + bool(case.get("correct", False)) + ) + if not case.get("correct", False): + status = (case.get("grader_log") or {}).get("status", case.get("status", "?")) + reasons[status] = reasons.get(status, 0) + 1 + + n = min(len(v) for v in by_problem.values()) + n_problems = len(by_problem) + + print(f"\n{'=' * 60}") + print(f" {eval_state.dataset_type}: {n_problems} problems, {n} sample(s) each") + print(f"{'=' * 60}") + + for k in sorted({1, 5, 10, 100, n}): + if k > n: + continue + score = sum( + pass_at_k(len(v), sum(v), k) for v in by_problem.values() + ) / n_problems + print(f" pass@{k:<4d} {score:.4f}") + + if n == 1: + solved = sum(1 for v in by_problem.values() if v[0]) + lo, hi = wilson_interval(solved, n_problems) + print(f" {solved}/{n_problems} solved 95% CI [{lo:.4f}, {hi:.4f}]") + + t_total = t_passed = 0 + for case in cases.values(): + log = case.get("grader_log") or {} + if "tests_total" in log: + t_total += log["tests_total"] + else: + try: + t_total += json.loads(case.get("expected") or "{}").get("n_tests", 0) + except ValueError: + pass + t_passed += log.get("tests_passed", 0) + if t_total: + print(f"\n test methods passed {t_passed}/{t_total} ({t_passed / t_total:.4f})") + + if reasons: + print("\n failures by cause:") + for status, count in sorted(reasons.items(), key=lambda kv: -kv[1]): + print(f" {status:<12s} {count}") + + def main(): parser = argparse.ArgumentParser( description="Simplified evaluation tool for llama.cpp" @@ -1358,13 +2379,28 @@ def main(): default="", help="Comma-separated display names for servers (default: use URLs)" ) + parser.add_argument( + "--api-key", + type=str, + default=None, + help="Bearer token for the eval servers: one key for all of them, or a " + "comma-separated key per --server (default: $LLAMA_API_KEY, else none)" + ) parser.add_argument( "--dataset", type=str, default="aime", - choices=["aime", "aime2025", "aime2026", "gsm8k", "gpqa"], + choices=["aime", "aime2025", "aime2026", "gsm8k", "gpqa", "humaneval", + "classeval", "agentic"], help="Dataset type (default: aime)" ) + parser.add_argument( + "--dataset-source", + type=str, + default=None, + help="Override the dataset location: a local .jsonl path or a HuggingFace " + "repo id (e.g. a patched HumanEval). Default: the suite's canonical repo" + ) parser.add_argument( "--n_cases", type=int, @@ -1407,6 +2443,25 @@ def main(): default=None, help="Min P sampling (default: not passed)" ) + parser.add_argument( + "--retries", + type=int, + default=5, + help="Times to resend a request that errors or comes back empty before " + "recording the case as a server error (default: 5)" + ) + parser.add_argument( + "--retry-delay", + type=float, + default=10.0, + help="Seconds to wait before the first resend; doubles on each further one (default: 10)" + ) + parser.add_argument( + "--reasoning-effort", + type=str, + default=None, + help="Reasoning effort sent as reasoning_effort with every request, e.g. low/medium/high (default: not passed)" + ) parser.add_argument( "--threads", type=str, @@ -1433,9 +2488,37 @@ def main(): parser.add_argument( "--grader-type", type=str, - default="llm", - choices=["regex", "cli", "llm"], - help="Grader type: regex, cli, or llm (default: llm)" + default=None, + choices=["regex", "cli", "llm", "exec"], + help="Grader type: regex, cli, llm, or exec (default: exec for code suites, " + "llm otherwise). 'exec' runs the generated code against the suite tests " + "in a sandboxed venv" + ) + parser.add_argument( + "--exec-timeout", + type=float, + default=None, + help="Per-problem wall-clock limit for the exec grader, in seconds " + "(default: per-suite, 15s for humaneval, 30s for classeval)" + ) + parser.add_argument( + "--exec-seed", + type=int, + default=0, + help="Seed the global RNG before running each test, so suites whose tests " + "draw random inputs score reproducibly (default: 0). Use -1 to leave " + "the RNG unseeded, matching the stock upstream harness" + ) + parser.add_argument( + "--sandbox-python", + type=str, + default=None, + help="Base interpreter used to build the execution venv (default: this one)" + ) + parser.add_argument( + "--rebuild-sandbox", + action="store_true", + help="Discard and re-provision the cached execution venv" ) parser.add_argument( "--grader-script", @@ -1455,6 +2538,51 @@ def main(): default="", help="Model name for LLM grader (default: same as main model)" ) + parser.add_argument( + "--grader-api-key", + type=str, + default=None, + help="Bearer token for the LLM grader server (default: the first " + "--api-key when the grader runs on the main server, else none)" + ) + parser.add_argument( + "--agentic-lang", + type=str, + default=None, + choices=["python", "typescript"], + help="Restrict the agentic suite to one language. Only that language's " + "toolchain is then provisioned." + ) + parser.add_argument( + "--agentic-max-turns", + type=int, + default=50, + help="Maximum assistant turns per agentic episode (default: 50)" + ) + parser.add_argument( + "--agentic-max-tokens", + type=int, + default=None, + help="Cap on generated tokens per agentic turn (default: no cap). " + "Bounds a model that falls into a repetition loop, but also cuts " + "off long reasoning; truncated turns are counted in the results." + ) + parser.add_argument( + "--agentic-context-limit", + type=int, + default=None, + help="Abandon an agentic episode once its context exceeds this many " + "tokens (default: no limit)" + ) + parser.add_argument( + "--agentic-transcripts", + type=Path, + default=None, + metavar="DIR", + help="Save each agentic episode's full message history to " + "DIR/.json, with a readable DIR/.html showing " + "every reasoning block, reply, tool call and tool result" + ) parser.add_argument( "--resume", action="store_true", @@ -1463,6 +2591,23 @@ def main(): args = parser.parse_args() + global DATASET_SOURCE, AGENTIC_OPTS + DATASET_SOURCE = args.dataset_source + AGENTIC_OPTS = { + "lang": args.agentic_lang, + "max_turns": args.agentic_max_turns, + "context_limit": args.agentic_context_limit, + "max_tokens": args.agentic_max_tokens, + } + + if args.grader_type is None: + args.grader_type = "exec" if args.dataset in CODE_SUITES else "llm" + + if args.grader_type == "exec" and args.dataset not in CODE_SUITES: + print(f"Error: --grader-type exec is only meaningful for code suites " + f"({', '.join(sorted(CODE_SUITES))}), not '{args.dataset}'") + sys.exit(1) + # Parse server URLs and thread counts server_urls = [u.strip() for u in args.server.split(",") if u.strip()] thread_counts = [int(t.strip()) for t in args.threads.split(",") if t.strip()] @@ -1480,11 +2625,30 @@ def main(): else: server_names = server_urls # fallback to URLs + api_key_arg = args.api_key if args.api_key is not None else os.environ.get("LLAMA_API_KEY") + if api_key_arg: + api_keys = [k.strip() for k in api_key_arg.split(",")] + if len(api_keys) == 1: + api_keys = api_keys * len(server_urls) + elif len(api_keys) != len(server_urls): + print(f"Error: --api-key ({len(api_keys)} keys) must be a single key or " + f"match --server ({len(server_urls)} URLs)") + sys.exit(1) + else: + api_keys = [None] * len(server_urls) + server_configs = [ - ServerConfig(url=url, threads=threads, name=name) - for url, threads, name in zip(server_urls, thread_counts, server_names) + ServerConfig(url=url, threads=threads, name=name, api_key=key) + for url, threads, name, key in zip(server_urls, thread_counts, server_names, api_keys) ] + if args.grader_api_key is not None: + grader_api_key = args.grader_api_key + else: + grader_url = args.grader_server or server_configs[0].url + grader_api_key = next( + (s.api_key for s in server_configs if s.url == grader_url), None) + if args.dataset == "gpqa" and args.grader_type != "llm": print("Error: GPQA dataset requires --grader-type llm") parser.print_help() @@ -1529,6 +2693,7 @@ def main(): grader_script=args.grader_script, grader_model_name=grader_model_name, grader_server_url=grader_server_url, + grader_api_key=grader_api_key, dataset_type=eval_state.dataset_type ) resume = True @@ -1548,6 +2713,7 @@ def main(): grader_script=args.grader_script, grader_model_name=grader_model_name, grader_server_url=grader_server_url, + grader_api_key=grader_api_key, dataset_type=args.dataset ) @@ -1563,6 +2729,8 @@ def main(): sampling_config["top_p"] = args.top_p if args.min_p is not None: sampling_config["min_p"] = args.min_p + if args.reasoning_effort is not None: + sampling_config["reasoning_effort"] = args.reasoning_effort eval_state = EvalState( dataset_type=args.dataset, @@ -1577,14 +2745,53 @@ def main(): eval_state.print_all_tasks() + if args.grader_type == "exec": + dataset = eval_state.loaded_dataset + + if dataset.is_agentic: + dataset.prepare(base_python=args.sandbox_python, + force=args.rebuild_sandbox) + else: + base_python = args.sandbox_python + want = dataset.preferred_python() + if not base_python and want: + base_python = resolve_python(want) + if base_python: + print(f"[sandbox] using Python {want} for " + f"{eval_state.dataset_type} ({base_python})") + + dataset.configure_python(base_python or sys.executable) + + sandbox = Sandbox( + name=eval_state.dataset_type, + packages=dataset.required_packages(), + provision=dataset.provision_steps(), + env=dataset.sandbox_env(), + base_python=base_python, + ) + sandbox.ensure(force=args.rebuild_sandbox) + preflight_sandbox(sandbox) + grader.sandbox = sandbox + grader.dataset = dataset + grader.exec_timeout = (args.exec_timeout if args.exec_timeout is not None + else dataset.default_exec_timeout()) + grader.exec_seed = None if args.exec_seed < 0 else args.exec_seed + processor = Processor( server_configs=server_configs, grader=grader, model_name=args.model, - n_predict=args.n_predict + n_predict=args.n_predict, + retries=args.retries, + retry_delay=args.retry_delay, + transcript_dir=args.agentic_transcripts, ) processor.evaluate(eval_state, verbose=args.verbose, resume=resume) + + if args.grader_type == "exec": + print_pass_at_k(eval_state) + print(f"\nEval state dumped to {args.output}") if __name__ == "__main__": diff --git a/examples/llama-eval/llama-server-simulator.py b/examples/llama-eval/llama-server-simulator.py index e64ba89335d9..dbd46d67f476 100755 --- a/examples/llama-eval/llama-server-simulator.py +++ b/examples/llama-eval/llama-server-simulator.py @@ -13,8 +13,6 @@ from dataclasses import dataclass from pathlib import Path -import datasets - # Set cache directory for HuggingFace datasets cache_dir = Path.home() / ".cache" / "huggingface" / "datasets" cache_dir.mkdir(parents=True, exist_ok=True) @@ -76,6 +74,8 @@ def _get_question_text(self, question: Dict) -> str: return question.get("problem", question.get("question", "")) def _load_dataset(self): + import datasets + if self.dataset_type == "aime": print(f"Loading AIME dataset (split: {self.split})...") cache_path = Path.home() / ".cache" / "huggingface" / "datasets" / "AI-MO___aimo-validation-aime" / "default" / "0.0.0" @@ -164,6 +164,58 @@ def get_answer(self, question: Dict) -> str: return str(normalized) if normalized is not None else answer return str(answer) +class CodeDataset: + """Stand-in for a code model: replays canonical solutions as chat replies. + + Handles both code suites: HumanEval keys off the function stub, ClassEval + off the class skeleton. + """ + + DEFAULT_SOURCE = { + "humaneval": "openai/openai_humaneval", + "classeval": "FudanSELab/ClassEval", + } + + def __init__(self, dataset_type: str, source: Optional[str] = None): + self.dataset_type = dataset_type + self.source = source or self.DEFAULT_SOURCE[dataset_type] + self.questions: List[Dict] = [] + self._load_dataset() + + def _load_dataset(self): + print(f"Loading {self.dataset_type} dataset (source: {self.source})...") + local = Path(self.source) + if local.exists(): + text = local.read_text() + if local.suffix == ".json": + rows = json.loads(text) + else: + rows = [json.loads(ln) for ln in text.splitlines() if ln.strip()] + else: + import datasets + rows = [dict(r) for r in datasets.load_dataset(self.source, split="test")] + self.questions = rows + print(f"{self.dataset_type} dataset loaded: {len(self.questions)} questions") + + def _stub_field(self) -> str: + return "skeleton" if self.dataset_type == "classeval" else "prompt" + + def _get_question_text(self, question: Dict) -> str: + return question[self._stub_field()] + + def find_question(self, request_text: str) -> Optional[Dict]: + for question in self.questions: + stub = question[self._stub_field()].rstrip() + if stub and stub in request_text: + return question + return None + + def get_answer(self, question: Dict) -> str: + if self.dataset_type == "classeval": + return question["solution_code"] + return question["prompt"] + question["canonical_solution"] + + class Simulator: def __init__( self, @@ -171,12 +223,17 @@ def __init__( host: str = "localhost", success_rate: float = 0.8, dataset_split: str = "train", - dataset_type: str = "aime" + dataset_type: str = "aime", + dataset_source: Optional[str] = None ): self.port = port self.host = host self.success_rate = success_rate - self.dataset = AimeDataset(dataset_split, dataset_type) + self.dataset_type = dataset_type + if dataset_type in ("humaneval", "classeval"): + self.dataset = CodeDataset(dataset_type, dataset_source) + else: + self.dataset = AimeDataset(dataset_split, dataset_type) self.eval_state = EvalState( id=dataset_type, tasks=[dataset_type], @@ -196,6 +253,12 @@ def _generate_response( else: response_text = self._generate_wrong_answer(question) + if self.dataset_type in ("humaneval", "classeval"): + response_text = ( + f"Here is the implementation:\n\n```python\n{response_text}\n```\n\n" + "Let me know if you want it optimised further." + ) + comp_tokens = random.randint(10000, 60000) tps_gen = random.uniform(90.0, 110.0) t_gen_ms = comp_tokens / tps_gen * 1000 @@ -229,6 +292,12 @@ def _generate_response( def _generate_wrong_answer(self, question: Dict) -> str: expected_answer = self.dataset.get_answer(question) + if self.dataset_type == "classeval": + return re.sub(r"(\n\s+def [^\n]+:\n)", r"\1 return None\n", + question["skeleton"]) + if self.dataset_type == "humaneval": + return question["prompt"] + " return None\n" + if expected_answer.isdigit(): wrong_answer = str(int(expected_answer) + 1) else: @@ -335,7 +404,7 @@ def main(): "--dataset", type=str, default="aime", - choices=["aime", "aime2025"], + choices=["aime", "aime2025", "humaneval", "classeval"], help="Dataset type (default: aime)" ) parser.add_argument( @@ -344,6 +413,12 @@ def main(): default="train", help="AIME dataset split to use (default: train)" ) + parser.add_argument( + "--dataset-source", + type=str, + default=None, + help="Override the dataset location (local .jsonl path or HuggingFace repo id)" + ) args = parser.parse_args() @@ -353,7 +428,8 @@ def main(): host=args.host, success_rate=args.success_rate, dataset_split=args.dataset_split, - dataset_type=args.dataset + dataset_type=args.dataset, + dataset_source=args.dataset_source ) server = HTTPServer((args.host, args.port), RequestHandler) From 2a667032dc788c3a071d538e0b572e45a3f12731 Mon Sep 17 00:00:00 2001 From: aiqubits Date: Fri, 16 Oct 2026 21:17:30 +0800 Subject: [PATCH 184/431] convert : respect Mistral format for DeepseekV2Model (#30669) Use the instance's is_mistral_format flag when DeepseekV2Model reloads configuration, allowing Mistral MoE checkpoints to read params.json. Fixes #29899 Signed-off-by: aiqubits --- conversion/deepseek.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conversion/deepseek.py b/conversion/deepseek.py index 817eb76128b6..946de53476ea 100644 --- a/conversion/deepseek.py +++ b/conversion/deepseek.py @@ -243,7 +243,7 @@ class DeepseekV2Model(TextModel): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - hparams: dict = ModelBase.load_hparams(self.dir_model, is_mistral_format=False) + hparams: dict = ModelBase.load_hparams(self.dir_model, is_mistral_format=self.is_mistral_format) self.origin_hf_arch = hparams.get('architectures', [None])[0] # special handling for Deepseek OCR From 7182ef71e6d5dd0c8ea76486fac134fcd25025d4 Mon Sep 17 00:00:00 2001 From: texasich <101962694+texasich@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:01:37 -0500 Subject: [PATCH 185/431] common: honor HTTP_PROXY/HTTPS_PROXY env vars in http client (#21752) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * common: honor HTTP_PROXY/HTTPS_PROXY env vars in http client common_http_client() was ignoring standard proxy env vars, causing downloads via -hf and similar flags to fail silently in environments behind an HTTP proxy (corporate networks, air-gapped infra, etc.). Checks HTTPS_PROXY/https_proxy for https:// URLs and HTTP_PROXY/http_proxy for http:// URLs, in that order. If a proxy URL is set and parseable, passes it to httplib's set_proxy() and set_proxy_basic_auth() as appropriate. Malformed proxy URLs are silently ignored so existing callers aren't affected. Fixes #21694 * common: drop misleading comment about generic proxy fallback The comment referenced a 'generic one' that doesn't exist in the code — both HTTP_PROXY and HTTPS_PROXY are scheme-specific. Reviewer noted the mismatch. Addresses review feedback from @ServeurpersoCom. * common : warn when the proxy URL from the environment is malformed * common : honor NO_PROXY when selecting the http proxy * ci : retrigger checks (test-thread-safety flake) * common : bypass the proxy for loopback destinations llama cli talks to its spawned server on 127.0.0.1 through common_http_client, so with HTTP_PROXY set and no NO_PROXY the health polling went to the corporate proxy, which rejects loopback targets, and the wait loop never exits ("Loading model..." hangs forever). A remote proxy can never reach the client machine's own loopback, so these destinations are now always contacted directly, before NO_PROXY is consulted: localhost, *.localhost, 127.0.0.0/8, ::1 and 0.0.0.0. Similar to Go's net/http. Proxy handling for downloads and remote --server-base endpoints is unchanged. Signed-off-by: texasich <101962694+texasich@users.noreply.github.com> * docs: move proxy tip from README to server docs per review: the top level README stays lean; the proxy howto now lives with the server docs under Usage Signed-off-by: texasich <101962694+texasich@users.noreply.github.com> --------- Signed-off-by: texasich <101962694+texasich@users.noreply.github.com> Co-authored-by: texasich Co-authored-by: texasich Co-authored-by: marecabo <23156476+marecabo@users.noreply.github.com> --- common/http.h | 141 +++++++++++++++++++++++++++++++++++++++++ tools/server/README.md | 4 ++ 2 files changed, 145 insertions(+) diff --git a/common/http.h b/common/http.h index 878ad1ce28a4..10d017f17ed5 100644 --- a/common/http.h +++ b/common/http.h @@ -1,5 +1,7 @@ #pragma once +#include "log.h" + #include #ifdef _WIN32 @@ -97,6 +99,118 @@ static common_http_url common_http_parse_url(const std::string & url) { return parts; } +// match a loopback destination: a remote proxy can never reach the client machine's own +// loopback, so these hosts are always contacted directly, before NO_PROXY is consulted +// (similar to Go's net/http) +static bool common_http_is_loopback(const std::string & host) { + auto lower = [](std::string s) { + for (auto & c : s) { + if (c >= 'A' && c <= 'Z') { + c += 'a' - 'A'; + } + } + return s; + }; + + const std::string h = lower(host); + + if (h == "localhost" || h == "::1" || h == "0.0.0.0") { + return true; + } + + if (h.size() > 10 && h.compare(h.size() - 10, 10, ".localhost") == 0) { + return true; + } + + // IPv4 literal: exactly four all-digit octets with values 0-255, loopback is 127.0.0.0/8 + int octets[4] = { 0, 0, 0, 0 }; + size_t pos = 0; + int n = 0; + bool complete = false; + while (n < 4 && pos <= h.size()) { + size_t dot = h.find('.', pos); + if (dot == std::string::npos) { + dot = h.size(); + } + if (dot == pos || dot - pos > 3) { + return false; + } + int val = 0; + for (size_t i = pos; i < dot; i++) { + if (h[i] < '0' || h[i] > '9') { + return false; + } + val = val * 10 + (h[i] - '0'); + } + if (val > 255) { + return false; + } + octets[n++] = val; + if (dot == h.size()) { + complete = (n == 4); + break; + } + pos = dot + 1; + } + + return complete && octets[0] == 127; +} + +// match a host against a NO_PROXY list: comma separated entries, "*" bypasses the proxy for every host, +// an entry matches the host itself or any of its subdomains, a leading dot is optional, ports are ignored +static bool common_http_proxy_bypass(const std::string & no_proxy, const std::string & host) { + auto lower = [](std::string s) { + for (auto & c : s) { + if (c >= 'A' && c <= 'Z') { + c += 'a' - 'A'; + } + } + return s; + }; + + const std::string h = lower(host); + + for (size_t pos = 0; pos < no_proxy.size(); ) { + size_t end = no_proxy.find(',', pos); + if (end == std::string::npos) { + end = no_proxy.size(); + } + + std::string entry = lower(no_proxy.substr(pos, end - pos)); + pos = end + 1; + + entry.erase(0, entry.find_first_not_of(" \t")); + auto last = entry.find_last_not_of(" \t"); + if (last == std::string::npos) { + continue; + } + entry.erase(last + 1); + + if (entry == "*") { + return true; + } + + if (entry.front() == '.') { + entry.erase(0, 1); + } + + if (entry.empty()) { + continue; + } + + if (h == entry) { + return true; + } + + if (h.size() > entry.size() && h[h.size() - entry.size() - 1] == '.' && + h.compare(h.size() - entry.size(), entry.size(), entry) == 0) { + return true; + } + } + + return false; +} + static std::pair common_http_client(const std::string & url) { common_http_url parts = common_http_parse_url(url); @@ -123,6 +237,33 @@ static std::pair common_http_client(const std: cli.set_follow_location(true); + // Honor HTTP_PROXY / HTTPS_PROXY environment variables (both lower- and upper-case). + auto getenv_s = [](const char * upper, const char * lower) -> std::string { + const char * val = std::getenv(upper); + if (!val) { + val = std::getenv(lower); + } + return val ? val : ""; + }; + + const std::string proxy_url = (parts.scheme == "https") + ? getenv_s("HTTPS_PROXY", "https_proxy") + : getenv_s("HTTP_PROXY", "http_proxy"); + + if (!proxy_url.empty() && !common_http_is_loopback(parts.host) && + !common_http_proxy_bypass(getenv_s("NO_PROXY", "no_proxy"), parts.host)) { + try { + common_http_url proxy = common_http_parse_url(proxy_url); + cli.set_proxy(proxy.host, proxy.port); + if (!proxy.user.empty()) { + cli.set_proxy_basic_auth(proxy.user, proxy.password); + } + } catch (const std::exception & e) { + // fall back to a direct connection, the URL itself is not logged as it can carry credentials + LOG_WRN("%s: ignoring malformed proxy URL from the environment: %s\n", __func__, e.what()); + } + } + return { std::move(cli), std::move(parts) }; } diff --git a/tools/server/README.md b/tools/server/README.md index 4e4472ec24bd..7d697633edac 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -405,6 +405,10 @@ Recommended `--cors-origins` setting, depending on where the server runs: Related flags: `--cors-origins`, `--cors-methods`, `--cors-headers`, `--cors-credentials` / `--no-cors-credentials`. Background and rationale: [#25655](https://github.com/ggml-org/llama.cpp/pull/25655). +### Running behind a proxy + +If your host reaches the internet through a corporate proxy, export `HTTP_PROXY` and `HTTPS_PROXY` before running commands that download models (e.g. `export HTTPS_PROXY=http://proxy.example.com:8080`). Hosts listed in `NO_PROXY` and loopback addresses are contacted directly. + ## Build `llama-server` is built alongside everything else from the root of the project From 23f7aabc16c72f6f707dba227251c35d7bd1ed96 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Fri, 16 Oct 2026 12:00:41 -0600 Subject: [PATCH 186/431] model : Granite Switch Decision (#30676) * feat(convert): Add decision.chat to support decision models that can also chat Branch: GraniteSwitchDecision AI-usage: full (Claude + Sonnet 5.5) Signed-off-by: Gabe Goodhart * feat(convert): Add conversion for Granite Switch as decision model The template follows from the usage sample: https://huggingface.co/ibm-granite/granite-switch-4.1-8b#guardian-and-decision-side-by-side Branch: GraniteSwitchDecision AI-usage: draft (Claude + Sonnet 5.5) Signed-off-by: Gabe Goodhart * feat(common): Add support for chat+decision models in common Branch: GraniteSwitchDecision AI-usage: full (Claude + Sonnet 5.5) Signed-off-by: Gabe Goodhart * feat(common): Add granite switch decision model enum Branch: GraniteSwitchDecision AI-usage: full (Claude + Sonnet 5.5) Signed-off-by: Gabe Goodhart * feat(server): Plumb through decision_chat bool to support decision+chat models Branch: GraniteSwitchDecision AI-usage: full (Claude + Sonnet 5.5) Signed-off-by: Gabe Goodhart * feat(server): Add Granite Switch decision support Branch: GraniteSwitchDecision AI-usage: draft (Claude + Sonnet 5.5) Signed-off-by: Gabe Goodhart * docs: Update server README for granite switch decisions / chat+decisions Branch: GraniteSwitchDecision AI-usage: draft (Claude + Sonnet 5.5) Signed-off-by: Gabe Goodhart * Revert "feat(convert): Add decision.chat to support decision models that can also chat" This reverts commit 952ef567647e53ac9beda7a1f79d6927240d068d. * Revert "feat(common): Add support for chat+decision models in common" This reverts commit 93ed782e3a8b2fd5968732c6525c7ad29ad4bea2. * Revert "feat(server): Plumb through decision_chat bool to support decision+chat models" This reverts commit 97e9af60ddaccbd82f1eb0a488adff43b16ee095. * fix: If Granite Switch supports decisions, it always supports text too Branch: GraniteSwitchDecision AI-usage: none Signed-off-by: Gabe Goodhart * docs: Remove reference to decision.chat in server README Branch: GraniteSwitchDecision AI-usage: none Signed-off-by: Gabe Goodhart --- common/common.cpp | 1 + common/common.h | 1 + conversion/granite.py | 31 ++++++++++++++++++++++++++++++- gguf-py/gguf/constants.py | 1 + tools/server/README.md | 4 ++-- tools/server/server-common.cpp | 2 ++ tools/server/server-decision.cpp | 4 ++-- tools/server/server-decision.h | 1 + 8 files changed, 40 insertions(+), 5 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index c5db228329f5..b59e66e3bd3f 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1174,6 +1174,7 @@ static const std::map COMMON_DECISION_TYPE_NA { COMMON_DECISION_TYPE_VLLM_D3, "vllm-d3" }, { COMMON_DECISION_TYPE_LFM2_D1, "lfm2-d1" }, { COMMON_DECISION_TYPE_LFM2_D1_OMNI, "lfm2-d1-omni" }, + { COMMON_DECISION_TYPE_GRANITE_SWITCH, "granite-switch"}, // for future models, ALWAYS add org name prefix to avoid name collisions, e.g. "orgname-modelname" }; diff --git a/common/common.h b/common/common.h index 2cd6e2ec9858..0c2d5e46f105 100644 --- a/common/common.h +++ b/common/common.h @@ -970,6 +970,7 @@ enum common_decision_type { COMMON_DECISION_TYPE_VLLM_D3, // same as pplx-decider, non-causal attention COMMON_DECISION_TYPE_LFM2_D1, // same as openjev, the labels depend on the question type COMMON_DECISION_TYPE_LFM2_D1_OMNI, // same as laya, other prompt layout + COMMON_DECISION_TYPE_GRANITE_SWITCH, // same as openjev, run by the decision aLoRA of granite switch COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported }; diff --git a/conversion/granite.py b/conversion/granite.py index 4cbeb7617892..d2ef7c836c8d 100644 --- a/conversion/granite.py +++ b/conversion/granite.py @@ -8,7 +8,7 @@ if TYPE_CHECKING: from torch import Tensor -from .base import MmprojModel, ModelBase, gguf, logger +from .base import MmprojModel, ModelBase, gguf, jinja_str_or_json, logger from .llama import LlamaModel from .mamba import Mamba2Model @@ -284,10 +284,39 @@ def set_gguf_parameters(self): self.gguf_writer.add_adapter_lora_rank(self._max_lora_rank) self.gguf_writer.add_adapter_token_ids_activate(self.hparams["adapter_token_ids"]) self.gguf_writer.add_adapter_token_ids_substitute(self.hparams["adapter_substitute_token_ids"]) + if self._has_decision_adapter(): + self.gguf_writer.add_decision_type(gguf.DecisionType.GRANITE_SWITCH) router_gain = float(self.hparams.get("control_token_gain", 15.0)) self.gguf_writer.add_adapter_router_gain(router_gain) logger.info("gguf: (graniteswitch) num_adapters=%s max_lora_rank=%s n_slots=%s router_gain=%s", self._n_adapters, self._max_lora_rank, self._n_slots, router_gain) + def _has_decision_adapter(self) -> bool: + return "decision" in self.hparams.get("adapter_names", []) + + def set_vocab(self): + super().set_vocab() + if self._has_decision_adapter(): + self.gguf_writer.add_chat_template([{"name": "systemone", "template": self._systemone_template()}]) + + def _systemone_template(self) -> str: + # prompt of the decision aLoRA: a state turn, then a question turn that opens with the control token + # (it replaces the first token of "### DECISION"), the answer is the label token that comes next + description = jinja_str_or_json("o.description") + option = ( + "{% if type != 'noul' %}{{ letters[loop.index0] }}: {% if o.description %}" + description + "{% endif %}" + "{% elif o.key == 'true' %}A: {% if o.description %}" + description + "{% else %}The statement is true.{% endif %}" + "{% else %}B: {% if o.description %}" + description + "{% else %}The statement is false.{% endif %}{% endif %}" + ) + return ( + "{% set letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' %}" + "<|start_of_role|>user<|end_of_role|>State:\n" + jinja_str_or_json("state") + "<|end_of_text|>\n" + "<|start_of_role|>user<|end_of_role|><|decision|> DECISION\nQuestion:\n" + jinja_str_or_json("instructions") + + "{% if type == 'score' %} Rate along the ordered levels below (lowest first).{% endif %}" + "{{ '\n\nOptions:\n' }}" + "{% for o in options %}" + option + "{{ '\n' }}{% endfor %}" + "{{ '\nReply with exactly one option label.<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|>' }}" + ) + def _lora_a(self, data: Tensor) -> Tensor: # on-disk A: [n_adapters, 1, max_rank, in] -> [n_adapters+1, max_rank, in] a = data.squeeze(1) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 00b51165c94b..4261e76e3e80 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -6283,6 +6283,7 @@ class DecisionType: VLLM_D3 = "vllm-d3" # same as pplx-decider, non-causal attention LFM2_D1 = "lfm2-d1" # same as openjev, the labels depend on the question type LFM2_D1_OMNI = "lfm2-d1-omni" # same head as laya on a bidirectional LFM2 trunk, other prompt layout + GRANITE_SWITCH = "granite-switch" # same as openjev, run by the decision aLoRA of granite switch class VisionProjectorType: diff --git a/tools/server/README.md b/tools/server/README.md index 7d697633edac..e4a0c418bf10 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -1723,9 +1723,9 @@ Follows the [TypeSafe API](https://docs.typesafe.ai/api), streaming is not suppo The questions of a request are answered independently, an answer does not depend on the other questions. The exception is clef: it reads all the questions in one prompt and decides them jointly. -The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with. +The number of options of a `choice` question is limited by the model, for example: 52 for openjev and granite-switch, 255 for laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with. -For laya, clef, vllm-d3 and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. A server that runs clef only serves this endpoint, text generation is not available. +For laya, clef, vllm-d3 and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. A server that runs clef only serves this endpoint, text generation is not available. A decision model that is also a text model, such as granite-switch, serves both and reports `["text", "decisions"]`. *Image input:* diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index c32c0d1a5aa7..5577c339dbc0 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -161,6 +161,8 @@ std::vector server_model_output_modalities(common_decision_type dec case COMMON_DECISION_TYPE_LFM2_D1: case COMMON_DECISION_TYPE_LFM2_D1_OMNI: return {"decisions"}; + case COMMON_DECISION_TYPE_GRANITE_SWITCH: + return {"text", "decisions"}; default: // fallback when there is no decision type or the metadata is bad return {"text"}; diff --git a/tools/server/server-decision.cpp b/tools/server/server-decision.cpp index b3c19e1f0304..3905c81b34c3 100644 --- a/tools/server/server-decision.cpp +++ b/tools/server/server-decision.cpp @@ -87,7 +87,7 @@ void server_decision_context::init(const llama_model * model) { bias[level] = std::strtof(val, nullptr); } - if (model_type == COMMON_DECISION_TYPE_OPENJEV) { + if (model_type == COMMON_DECISION_TYPE_OPENJEV || model_type == COMMON_DECISION_TYPE_GRANITE_SWITCH) { // one letter per option, each must be a single token const std::string letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; for (const char c : letters) { @@ -753,7 +753,7 @@ void server_decision_context::fill_task( const std::string prompt = render(state, questions, question, variant, files.size()); - if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3) { + if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_GRANITE_SWITCH || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3) { // lev reads the ratings of a noul question at its first labels, not at the digits task.decision.labels.assign(labels.begin(), labels.begin() + n_outputs(question)); } diff --git a/tools/server/server-decision.h b/tools/server/server-decision.h index bfe9fd0ee518..ed97244293dc 100644 --- a/tools/server/server-decision.h +++ b/tools/server/server-decision.h @@ -39,6 +39,7 @@ struct server_decision_context { bool can_share_prompt() const { switch (type) { case COMMON_DECISION_TYPE_OPENJEV: + case COMMON_DECISION_TYPE_GRANITE_SWITCH: case COMMON_DECISION_TYPE_LEV: case COMMON_DECISION_TYPE_KEV: case COMMON_DECISION_TYPE_NIMBLE: From 01fc3aae846caa644f0cee360f108d4c48e4aaa1 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 16 Oct 2026 15:07:55 -0700 Subject: [PATCH 187/431] model: add Beam (Reflection) (#30670) --- common/chat.cpp | 7 + common/jinja/value.cpp | 9 ++ common/parsers/harmony-xml.cpp | 207 ++++++++++++++++++++++++++ common/parsers/parsers.h | 2 + common/parsers/sources.cmake | 1 + conversion/__init__.py | 1 + conversion/beam.py | 135 +++++++++++++++++ gguf-py/gguf/constants.py | 26 ++++ src/llama-arch.cpp | 1 + src/llama-arch.h | 1 + src/llama-graph.cpp | 26 +++- src/llama-graph.h | 7 +- src/llama-hparams.h | 2 + src/llama-model.cpp | 11 +- src/llama-model.h | 1 + src/llama-vocab.cpp | 5 +- src/models/beam.cpp | 261 +++++++++++++++++++++++++++++++++ src/models/models.h | 13 ++ tests/test-jinja.cpp | 6 + tests/test-llama-archs.cpp | 21 +++ 20 files changed, 732 insertions(+), 11 deletions(-) create mode 100644 common/parsers/harmony-xml.cpp create mode 100644 conversion/beam.py create mode 100644 src/models/beam.cpp diff --git a/common/chat.cpp b/common/chat.cpp index 118072b6b514..99ed74faf7a9 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1104,6 +1104,13 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const return common_chat_params_init_llm_jp_harmony; } + // Harmony XML tool calls, checked before the GPT-OSS JSON calls + if (src.find("<|channel|>") != std::string::npos && src.find("<|constrain|>xml") != std::string::npos && + src.find("rfl:parameter") != std::string::npos) { + LOG_DBG("Using specialized template: Harmony XML\n"); + return common_chat_params_init_harmony_xml; + } + // GPT-OSS - has unique channel-based structure that needs dedicated handler if (src.find("<|channel|>") != std::string::npos) { LOG_DBG("Using specialized template: GPT-OSS\n"); diff --git a/common/jinja/value.cpp b/common/jinja/value.cpp index 0ecbbf8a8f7f..a1dc9691b983 100644 --- a/common/jinja/value.cpp +++ b/common/jinja/value.cpp @@ -1299,6 +1299,15 @@ const func_builtins & value_object_t::get_builtins() const { } return result; }}, + {"list", [](const func_args & args) -> value { + args.ensure_vals(); + const auto & obj = args.get_pos(0)->as_ordered_object(); + auto result = mk_val(); + for (const auto & pair : obj) { + result->push_back(pair.first); + } + return result; + }}, {"values", [](const func_args & args) -> value { args.ensure_vals(); const auto & obj = args.get_pos(0)->as_ordered_object(); diff --git a/common/parsers/harmony-xml.cpp b/common/parsers/harmony-xml.cpp new file mode 100644 index 000000000000..bf05fac5d680 --- /dev/null +++ b/common/parsers/harmony-xml.cpp @@ -0,0 +1,207 @@ +#include "parsers.h" + +// Harmony with XML tool calls: <|start|>assistant to=NAME<|channel|>analysis <|constrain|>xml<|message|>\nVALUE\n +common_chat_params common_chat_params_init_harmony_xml(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" }, + { COMMON_CHAT_ROLE_USER, "<|start|>user" }, + { COMMON_CHAT_ROLE_SYSTEM, "<|start|>developer" }, + { COMMON_CHAT_ROLE_SYSTEM, "<|start|>system" }, + }; + + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = true; + + data.thinking_start_tag = "<|channel|>analysis<|message|>"; + data.thinking_end_tags = {"<|end|>"}; + + // EOG text must reach the parser: an XML call has no closing tag of its own + static const std::vector EOG = { + "<|call|>", "<|return|>", "<|endoftext|>", "<|im_end|>", "<|fim_pad|>", "<|repo_name|>", "<|file_sep|>", + }; + + data.preserved_tokens = { + "<|channel|>", "<|constrain|>", "<|message|>", "<|start|>", "<|end|>", + }; + data.preserved_tokens.insert(data.preserved_tokens.end(), EOG.begin(), EOG.end()); + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = "<|start|>assistant<|channel|>analysis<|message|>" + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += "<|end|><|start|>assistant<|channel|>final<|message|>" + msg.render_content(); + } + + data.prompt.append(data.generation_prompt); + } + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); + auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + + static const std::string PARAM_OPEN = "assistant")); + + auto eog = p.choice(); + for (const auto & tok : EOG) { + eog |= p.literal(tok); + } + auto with_eog = [&](std::vector delims) { + delims.insert(delims.end(), EOG.begin(), EOG.end()); + return delims; + }; + + auto analysis_open = p.literal("<|channel|>analysis<|message|>"); + auto analysis_body = p.until_one_of(with_eog({ "<|end|>", "<|constrain|><|start|>assistant", "<|start|>assistant" })); + auto analysis_close = p.optional(p.literal("<|end|>") | p.literal("<|constrain|>") | eog); + if (extract_reasoning) { + p.rule("analysis", analysis_open + p.reasoning(analysis_body) + analysis_close); + } else { + p.rule("analysis", p.atomic(p.content(analysis_open)) + p.content(analysis_body + analysis_close)); + } + auto analysis = p.ref("analysis"); + + auto final_msg = p.rule("final", + p.literal("<|channel|>final<|message|>") + + p.content(p.until_one_of(with_eog({ "<|end|>" }))) + + p.optional(p.literal("<|end|>") | eog)); + + if (has_response_format) { + auto constrain_type = p.chars("[A-Za-z0-9_-]", 1, -1); + auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type); + auto response_format = p.rule("response-format", + p.literal("<|channel|>final") + constraint + p.literal("<|message|>") + + p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) + + p.optional(p.literal("<|end|>") | eog)); + + return start + p.zero_or_more(analysis + start) + response_format; + } + + if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { + // tolerate a missing closing tag + std::vector value_ends = with_eog({ PARAM_OPEN, "<|end|>" }); + std::vector string_ends = { PARAM_CLOSE }; + auto next_open = p.choice(); + for (const auto & e : value_ends) { + string_ends.push_back("\n" + e); + next_open |= p.literal(e); + } + auto arg_close = p.tool_arg_close(p.gbnf( + p.literal(PARAM_CLOSE) | (p.literal("\n") + p.peek(p.optional(p.literal("\n")) + next_open)), + gbnf_format_literal("\n") + " " + gbnf_format_literal(PARAM_CLOSE.substr(1)) + "?")); + auto arg_string = p.rule("rfl-arg-string", + p.ac(p.tool_arg_string_value(p.until_one_of(string_ends)) + arg_close, PARAM_CLOSE)); + auto arg_sep = p.optional(p.literal("\n")); + + std::string call_close_gbnf = gbnf_format_literal("<|end|>"); + for (const auto & tok : EOG) { + call_close_gbnf += " | " + gbnf_format_literal(tok); + } + auto call_close = p.gbnf(p.literal("<|end|>") | eog | p.peek(p.literal("<|start|>assistant")), + "(" + call_close_gbnf + ")?"); + + auto tool_choice = p.choice(); + foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + + std::vector arg_rules; + foreach_parameter(function, [&](size_t param_index, const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) { + auto rule_name = "tool-" + std::to_string(tool_index) + "-arg-" + std::to_string(param_index); + auto arg_open = p.tool_arg_open(p.literal(PARAM_OPEN) + p.tool_arg_name(p.literal(param.name)) + p.literal("\">\n")); + + auto types = param.schema->value_types(); + auto arg_value = p.eps(); + if (!types.has(common_chat_schema::TYPE_STRING) && + (types.has(common_chat_schema::TYPE_OBJECT) || types.has(common_chat_schema::TYPE_ARRAY))) { + // the schema grammar fixes the key order, and the model writes keys in its own order + arg_value = p.tool_arg_json_value(p.json()) + arg_close; + } else if (!types.has(common_chat_schema::TYPE_STRING)) { + arg_value = p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", doc, *param.schema)) + arg_close; + } else if (types.is_only(common_chat_schema::TYPE_STRING)) { + arg_value = arg_string; + } else { + // grammar keeps only the string rule; the parser tries JSON first to type the value + auto json_value = p.choice(); + if (types.has(common_chat_schema::TYPE_OBJECT)) { + json_value |= p.json_object(); + } + if (types.has(common_chat_schema::TYPE_ARRAY)) { + json_value |= p.json_array(); + } + if (types.has(common_chat_schema::TYPE_NUMBER) || types.has(common_chat_schema::TYPE_INTEGER)) { + json_value |= p.json_number(); + } + if (types.has(common_chat_schema::TYPE_BOOLEAN)) { + json_value |= p.json_bool(); + } + if (types.has(common_chat_schema::TYPE_NULL)) { + json_value |= p.json_null(); + } + arg_value = p.gbnf(p.atomic(p.tool_arg_json_value(json_value) + arg_close) | arg_string, "rfl-arg-string"); + } + + arg_rules.push_back(p.rule(rule_name, p.tool_arg(arg_open + arg_value))); + }); + + if (arg_rules.empty()) { + auto any_name = p.chars("[^\"\\n<]", 1, -1); + arg_rules.push_back(p.rule("tool-" + std::to_string(tool_index) + "-arg-any", + p.tool_arg(p.tool_arg_open(p.literal(PARAM_OPEN) + p.tool_arg_name(any_name) + p.literal("\">\n")) + arg_string))); + } + + auto args = p.zero_or_more(p.choice(arg_rules) + arg_sep); + + auto call = p.tool( + p.tool_open(p.literal(" to=") + p.tool_name(p.literal(name)) + + p.literal("<|channel|>analysis <|constrain|>xml<|message|>")) + + p.tool_args(args) + + p.tool_close(call_close)); + + tool_choice |= p.rule("tool-" + std::to_string(tool_index), call); + }); + + auto tool_calls = inputs.parallel_tool_calls + ? tool_choice + p.zero_or_more(start + tool_choice) + : tool_choice; + auto tool_call = p.trigger_rule("tool-call", tool_calls); + + if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) { + return start + p.zero_or_more(analysis + start) + tool_call; + } + + return start + p.zero_or_more(analysis + start) + (tool_call | final_msg | p.content(p.rest())); + } + + auto unsolicited = p.rule("unsolicited", p.atomic(p.literal(" to=") + p.rest())); + + // output without a channel header is returned as content instead of failing the request + return start + p.zero_or_more(analysis + start) + (final_msg | unsolicited | p.content(p.rest())); + }); + + if (include_grammar) { + data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + data.parser.build_grammar(builder, data.grammar_lazy); + }); + + if (data.grammar_lazy) { + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^\\s+to$" }, + { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(\\s+to)" }, + }; + } + } + + return data; +} diff --git a/common/parsers/parsers.h b/common/parsers/parsers.h index fac69c096db1..532a44f014fe 100644 --- a/common/parsers/parsers.h +++ b/common/parsers/parsers.h @@ -59,6 +59,8 @@ common_chat_params common_chat_params_init_gigachat_v3(const common_chat_templat common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl, const autoparser::generation_params & inputs); +common_chat_params common_chat_params_init_harmony_xml(const common_chat_template & tmpl, const autoparser::generation_params & inputs); + common_chat_params common_chat_params_init_k2_horizon(const common_chat_template & tmpl, const autoparser::generation_params & inputs); common_chat_params common_chat_params_init_kimi_k2(const common_chat_template & tmpl, const autoparser::generation_params & inputs); diff --git a/common/parsers/sources.cmake b/common/parsers/sources.cmake index 5ee65b188a5b..2174f4519a64 100644 --- a/common/parsers/sources.cmake +++ b/common/parsers/sources.cmake @@ -9,6 +9,7 @@ set(LLAMA_CHAT_PARSERS_SOURCES ${CMAKE_CURRENT_LIST_DIR}/gemma4.cpp ${CMAKE_CURRENT_LIST_DIR}/gigachat-v3.cpp ${CMAKE_CURRENT_LIST_DIR}/gpt-oss.cpp + ${CMAKE_CURRENT_LIST_DIR}/harmony-xml.cpp ${CMAKE_CURRENT_LIST_DIR}/k2-horizon.cpp ${CMAKE_CURRENT_LIST_DIR}/kimi-k2.cpp ${CMAKE_CURRENT_LIST_DIR}/kimi-k3.cpp diff --git a/conversion/__init__.py b/conversion/__init__.py index c2683aec471c..01c8db29680c 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -24,6 +24,7 @@ "ArcticForCausalLM": "arctic", "AudioFlamingo3ForConditionalGeneration": "qwen", "BaiChuanForCausalLM": "baichuan", + "BeamForCausalLM": "beam", "BaichuanForCausalLM": "baichuan", "BailingMoeForCausalLM": "bailingmoe", "BailingMoeV2ForCausalLM": "bailingmoe", diff --git a/conversion/beam.py b/conversion/beam.py new file mode 100644 index 000000000000..67d0cc93ca63 --- /dev/null +++ b/conversion/beam.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +from typing import Callable, Iterable, TYPE_CHECKING + +import torch + +if TYPE_CHECKING: + from torch import Tensor + +from .base import ModelBase, TextModel, gguf, logger + + +@ModelBase.register("BeamForCausalLM") +class BeamModel(TextModel): + model_arch = gguf.MODEL_ARCH.BEAM + + # reference NoPE query-scale constants; not in config.json + NOPE_SCALE_POWER = 0.1 + NOPE_SCALE_LENGTH = 2048 + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + hp = self.hparams + rope = hp.get("rope_parameters") or {} + if rope.get("full_attention") is not None: + raise ValueError("Beam: expected NoPE full-attention layers (rope_parameters.full_attention = null)") + if "sliding_attention" in rope: + swa_rope = rope["sliding_attention"] or {} + else: + # flat rope_theta / rope_scaling, already normalized by TextModel + swa_rope = {k: v for k, v in self.rope_parameters.items() if k not in ("sliding_attention", "full_attention", "type")} + swa_rope.update(self.rope_parameters.get("sliding_attention") or {}) + if swa_rope.get("rope_type", "default") != "default": + raise ValueError(f"Beam: unsupported sliding-attention rope type {swa_rope.get('rope_type')!r}") + # full layers are NoPE, so mirror the sliding rope params for the base class + swa_rope = {k: v for k, v in swa_rope.items() if not (k == "rope_type" and v == "default")} + self.rope_parameters = {"sliding_attention": dict(swa_rope), "full_attention": dict(swa_rope)} + + layer_types = hp["layer_types"] + mlp_types = hp["mlp_layer_types"] + n_layer = hp["num_hidden_layers"] + if len(layer_types) != n_layer or len(mlp_types) != n_layer: + raise ValueError("Beam: layer_types / mlp_layer_types must have num_hidden_layers entries") + if any(t not in ("sliding_attention", "full_attention") for t in layer_types): + raise ValueError(f"Beam: unknown layer type in {layer_types}") + if "sliding_attention" not in layer_types: + raise ValueError("Beam: expected at least one sliding_attention layer") + n_dense = 0 + while n_dense < n_layer and mlp_types[n_dense] == "dense": + n_dense += 1 + if any(t != "sparse" for t in mlp_types[n_dense:]): + raise ValueError("Beam: llama.cpp supports dense MLP layers only as a leading block") + self._n_dense_lead = n_dense + if hp.get("attention_bias", False): + raise ValueError("Beam: attention_bias=True is not supported") + self._experts: dict[tuple[int, str], dict[int, Tensor]] = {} + + def set_vocab(self): + self._set_vocab_gpt2() + # otherwise EOT is guessed from a hash-ordered scan + from transformers import AutoTokenizer + tok = AutoTokenizer.from_pretrained(self.dir_model) + ret_id = tok.convert_tokens_to_ids("<|return|>") # ty: ignore[unresolved-attribute] + if isinstance(ret_id, int) and ret_id >= 0: + self.gguf_writer.add_eot_token_id(ret_id) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + hp = self.hparams + + head_dim = hp["head_dim"] + self.gguf_writer.add_rope_dimension_count(head_dim) + + self.gguf_writer.add_sliding_window(hp["sliding_window"]) + self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in hp["layer_types"]]) + + self.gguf_writer.add_attn_temperature_scale(self.NOPE_SCALE_POWER) + self.gguf_writer.add_attn_temperature_length(self.NOPE_SCALE_LENGTH) + + self.gguf_writer.add_leading_dense_block_count(self._n_dense_lead) + self.gguf_writer.add_expert_feed_forward_length(hp["moe_intermediate_size"]) + self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID) + self.gguf_writer.add_expert_weights_norm(bool(hp.get("norm_topk_prob", True))) + # equivalent to the reference's post-aggregation scaling + self.gguf_writer.add_expert_weights_scale(float(hp["routed_scaling_factor"])) + + if (softcap := hp.get("final_logit_softcapping")) is not None: + self.gguf_writer.add_final_logit_softcapping(float(softcap)) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + if name.endswith(".self_attn.gate_bias"): + name = name.replace(".self_attn.gate_bias", ".self_attn.gate_proj.bias") + return super().filter_tensors((name, gen)) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + hp = self.hparams + if name.endswith(".self_attn.gate_proj.bias"): + assert tuple(data_torch.shape) == (hp["num_attention_heads"],), f"{name}: one gate bias per head expected, got {tuple(data_torch.shape)}" + elif name.endswith(".self_attn.gate_proj.weight"): + assert tuple(data_torch.shape) == (hp["num_attention_heads"] * hp["head_dim"], hp["hidden_size"]), f"{name}: {tuple(data_torch.shape)}" + elif name.endswith(".mlp.gate.e_score_correction.bias"): + assert tuple(data_torch.shape) == (hp["num_experts"],), f"{name}: {tuple(data_torch.shape)}" + elif name.endswith((".self_attn.q_norm.weight", ".self_attn.k_norm.weight")): + assert tuple(data_torch.shape) == (hp["head_dim"],), f"{name}: per-head q/k norm expected, got {tuple(data_torch.shape)}" + elif bid is not None and bid < self._n_dense_lead and ".mlp.experts." in name: + raise ValueError(f"Beam: expert tensor {name} in a dense layer") + if ".mlp.experts." in name: + assert bid is not None + n_experts = self.hparams["num_experts"] + parts = name.split(".") + xid = int(parts[parts.index("experts") + 1]) + proj = parts[parts.index("experts") + 2] + if proj not in ("gate_proj", "up_proj", "down_proj"): + raise ValueError(f"Beam: unexpected expert tensor {name}") + bucket = self._experts.setdefault((bid, proj), {}) + if xid in bucket: + raise ValueError(f"Beam: duplicate expert tensor {name}") + bucket[xid] = data_torch + if len(bucket) < n_experts: + return + del self._experts[(bid, proj)] + data = torch.stack([bucket[e] for e in range(n_experts)], dim=0) + yield from super().modify_tensors(data, f"model.layers.{bid}.mlp.experts.{proj}.weight", bid) + return + + yield from super().modify_tensors(data_torch, name, bid) + + def prepare_tensors(self): + super().prepare_tensors() + if self._experts: + missing = sorted(f"layer {b} {p} ({len(v)} experts)" for (b, p), v in self._experts.items()) + raise ValueError(f"Beam: unprocessed experts: {missing}") + logger.info("Beam: all routed experts stacked") diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 4261e76e3e80..bfeab6adfde6 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -573,6 +573,7 @@ class MODEL_ARCH(IntEnum): QWEN3MOE = auto() QWEN3NEXT = auto() HRM_TEXT = auto() + BEAM = auto() QWEN3VL = auto() QWEN3VLMOE = auto() QWEN35 = auto() @@ -1388,6 +1389,7 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH.QWEN3MOE: "qwen3moe", MODEL_ARCH.QWEN3NEXT: "qwen3next", MODEL_ARCH.HRM_TEXT: "hrm_text", + MODEL_ARCH.BEAM: "beam", MODEL_ARCH.QWEN3VL: "qwen3vl", MODEL_ARCH.QWEN3VLMOE: "qwen3vlmoe", MODEL_ARCH.QWEN35: "qwen35", @@ -2981,6 +2983,30 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD, MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM, ], + MODEL_ARCH.BEAM: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_POST_NORM, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_Q_NORM, + MODEL_TENSOR.ATTN_K_NORM, + MODEL_TENSOR.ATTN_GATE, + MODEL_TENSOR.FFN_PRE_NORM, + MODEL_TENSOR.FFN_POST_NORM, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_EXP_PROBS_B, + MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP, + ], MODEL_ARCH.HRM_TEXT: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT, diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 1476a079a7cd..bef2d45c0ba2 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -165,6 +165,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_QWEN3TTS, "qwen3tts" }, { LLM_ARCH_POCKETTTS, "pockettts" }, { LLM_ARCH_K2_HORIZON, "k2-horizon" }, + { LLM_ARCH_BEAM, "beam" }, { LLM_ARCH_UNKNOWN, "(unknown)" }, }; diff --git a/src/llama-arch.h b/src/llama-arch.h index 0c842d312db1..c305c17663cb 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -170,6 +170,7 @@ enum llm_arch { LLM_ARCH_MINIMAX_01, LLM_ARCH_HRM_TEXT, LLM_ARCH_K2_HORIZON, + LLM_ARCH_BEAM, LLM_ARCH_UNKNOWN, }; diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 454c21404266..2bc3a291a456 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -218,17 +218,31 @@ void llm_graph_input_attn_temp::set_input(const llama_ubatch * ubatch) { GGML_ASSERT(n_attn_temp_floor_scale != 0); std::vector attn_scale_data(n_tokens, 0.0f); - for (int i = 0; i < n_tokens; ++i) { - const float pos = ubatch->pos[i]; - attn_scale_data[i] = std::log( - std::floor((pos + f_attn_temp_offset) / n_attn_temp_floor_scale) + 1.0 - ) * f_attn_temp_scale + 1.0; + if (attn_temp_pow) { + // (1 + (pos + offset) / length) ^ power + for (int i = 0; i < n_tokens; ++i) { + const double pos = ubatch->pos[i]; + attn_scale_data[i] = (float) std::exp((double) f_attn_temp_scale * + std::log1p((pos + f_attn_temp_offset) / (double) n_attn_temp_floor_scale)); + } + } else { + for (int i = 0; i < n_tokens; ++i) { + const float pos = ubatch->pos[i]; + attn_scale_data[i] = std::log( + std::floor((pos + f_attn_temp_offset) / n_attn_temp_floor_scale) + 1.0 + ) * f_attn_temp_scale + 1.0; + } } ggml_backend_tensor_set(attn_scale, attn_scale_data.data(), 0, n_tokens*ggml_element_size(attn_scale)); } } +bool llm_graph_input_attn_temp::can_reuse(const llm_graph_params & params) { + // only the power form opts in; the log form keeps no reuse + return attn_temp_pow && attn_scale != nullptr && attn_scale->ne[2] == params.ubatch.n_tokens; +} + void llm_graph_input_pos_bucket::set_input(const llama_ubatch * ubatch) { if (pos_bucket) { const int64_t n_tokens = ubatch->n_tokens; @@ -2743,7 +2757,7 @@ ggml_tensor * llm_graph_context::build_inp_pos() const { } ggml_tensor * llm_graph_context::build_inp_attn_scale() const { - auto inp = std::make_unique(hparams.n_attn_temp_floor_scale, hparams.f_attn_temp_scale, hparams.f_attn_temp_offset); + auto inp = std::make_unique(hparams.n_attn_temp_floor_scale, hparams.f_attn_temp_scale, hparams.f_attn_temp_offset, hparams.attn_temp_pow); auto & cur = inp->attn_scale; diff --git a/src/llama-graph.h b/src/llama-graph.h index 816fa07cc7c4..f66bedb3254f 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -188,17 +188,20 @@ class llm_graph_input_pos : public llm_graph_input_i { // temperature tuning, used by llama4 class llm_graph_input_attn_temp : public llm_graph_input_i { public: - llm_graph_input_attn_temp(uint32_t n_attn_temp_floor_scale, float f_attn_temp_scale, float f_attn_temp_offset) - : n_attn_temp_floor_scale(n_attn_temp_floor_scale), f_attn_temp_scale(f_attn_temp_scale), f_attn_temp_offset(f_attn_temp_offset) {} + llm_graph_input_attn_temp(uint32_t n_attn_temp_floor_scale, float f_attn_temp_scale, float f_attn_temp_offset, bool attn_temp_pow = false) + : n_attn_temp_floor_scale(n_attn_temp_floor_scale), f_attn_temp_scale(f_attn_temp_scale), f_attn_temp_offset(f_attn_temp_offset), attn_temp_pow(attn_temp_pow) {} virtual ~llm_graph_input_attn_temp() = default; void set_input(const llama_ubatch * ubatch) override; + bool can_reuse(const llm_graph_params & params) override; + ggml_tensor * attn_scale = nullptr; // F32 [n_batch] const uint32_t n_attn_temp_floor_scale; const float f_attn_temp_scale; const float f_attn_temp_offset; + const bool attn_temp_pow; }; class llm_graph_input_pos_bucket : public llm_graph_input_i { diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 3e025368b91f..719d7081dd21 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -280,6 +280,8 @@ struct llama_hparams { uint32_t n_attn_temp_floor_scale = 0; float f_attn_temp_scale = 0.0f; float f_attn_temp_offset = 0.0f; // offset position index + // use (1 + (pos + offset) / floor_scale) ^ scale instead of the llama4 log form + bool attn_temp_pow = false; // realtime models: number of tokens the output lags behind the audio input uint32_t n_tokens_delay = 0; diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 3da32333b10a..099c7a052f46 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -322,6 +322,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_minimax_m3(params); case LLM_ARCH_HRM_TEXT: return new llama_model_hrm_text(params); + case LLM_ARCH_BEAM: + return new llama_model_beam(params); case LLM_ARCH_COGVLM: return new llama_model_cogvlm(params); case LLM_ARCH_PANGU_EMBED: @@ -405,6 +407,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str static const std::regex pattern_idx_cache ("cache_idx_(k|v)_l\\d*"); static const std::regex pattern_dsv4_state ("dsv4_(csa|hca|lid)_state_(kv|score)_l\\d*"); static const std::regex pattern_attn_sinks ("blk\\.\\d*\\.attn_sinks.weight"); + static const std::regex pattern_attn_gate_bias ("blk\\.\\d*\\.attn_gate\\.bias"); static const std::regex pattern_attn_out_weight ("blk\\.\\d*\\.attn_output.weight"); static const std::regex pattern_attn_out_bias ("blk\\.\\d*\\.attn_output.bias"); static const std::regex pattern_attn_out_a_weight("blk\\.\\d*\\.attn_output_a\\.weight"); @@ -550,7 +553,8 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str if (std::regex_match(tensor_name, pattern_qk_norm)) { return get_tensor_config_impl(tensor->ne[1] == 1 ? GGML_BACKEND_SPLIT_AXIS_MIRRORED : GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight"); } - if (std::regex_match(tensor_name, pattern_kv_cache) || std::regex_match(tensor_name, pattern_attn_sinks)) { + if (std::regex_match(tensor_name, pattern_kv_cache) || std::regex_match(tensor_name, pattern_attn_sinks) || + std::regex_match(tensor_name, pattern_attn_gate_bias)) { return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output.weight"); } if (std::regex_match(tensor_name, pattern_attn_out_weight)) { @@ -749,6 +753,10 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str const int64_t granularity_q = std::lcm(n_embd_q, blck_size_perf); const int64_t granularity_head = granularity_q / hparams.n_embd_head_k(il); // for tensors with one value per head + if (std::regex_match(tensor_name, pattern_attn_gate_bias)) { + GGML_ASSERT(segments.size() == 1); + return {granularity_head}; + } if (std::regex_match(tensor_name, pattern_attn_sinks)) { GGML_ASSERT(segments.size() == 1); if (is_dsv4) { @@ -3606,6 +3614,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_MELLUM: case LLM_ARCH_MAPLE: case LLM_ARCH_HRM_TEXT: + case LLM_ARCH_BEAM: return LLAMA_ROPE_TYPE_NEOX; case LLM_ARCH_DFLASH: diff --git a/src/llama-model.h b/src/llama-model.h index 6335dc2bc318..6487b0e9c434 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -310,6 +310,7 @@ struct llama_layer { struct ggml_tensor * wv_enc = nullptr; struct ggml_tensor * wo_enc = nullptr; struct ggml_tensor * wqkv_gate = nullptr; + struct ggml_tensor * wqkv_gate_b = nullptr; // K2 Horizon MoVA struct ggml_tensor * attn_v_gate = nullptr; struct ggml_tensor * attn_v_gate_b = nullptr; diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index 9fd5b9ef3f2a..05d2527dc073 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -3142,9 +3142,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { } } - // gemma4 and plamo have a normal token, unlike paddleocr + // gemma4, plamo and beam have a normal token, unlike paddleocr { - bool has_normal_s_marker = false; + // beam's tool call marker is not an EOG token, so it is looked up in the whole vocab + bool has_normal_s_marker = token_to_id.count(" 0; bool has_s = false; llama_token s_id = LLAMA_TOKEN_NULL; diff --git a/src/models/beam.cpp b/src/models/beam.cpp new file mode 100644 index 000000000000..55817dd4839a --- /dev/null +++ b/src/models/beam.cpp @@ -0,0 +1,261 @@ +#include "models.h" + +void llama_model_beam::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); + ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm); + hparams.f_final_logit_softcapping = 0.0f; // absent key = no softcap (default is 30) + ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping, false); + + if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) { + hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID; + } + if (hparams.n_layer_dense_lead > hparams.n_layer()) { + throw std::runtime_error("beam: leading_dense_block_count exceeds the layer count"); + } + + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; // masked iff p1 - p0 >= n_swa + uint32_t n_pattern = 0; + ml.get_arr_n(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, n_pattern); + if (hparams.n_swa == 0 || n_pattern != hparams.n_layer_all) { + throw std::runtime_error("beam: requires a positive sliding window and a sliding window pattern for each layer"); + } + ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl); + if (!hparams.is_swa_any()) { + throw std::runtime_error("beam: sliding window pattern has no sliding window layer"); + } + + // full layers are NoPE + hparams.rope_pattern = hparams.is_swa_impl; + hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; + hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; + ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false); + + // NoPE query scale uses pos + 1 + hparams.f_attn_temp_scale = 0.1f; + hparams.attn_temp_length = 2048; + ml.get_key(LLM_KV_ATTENTION_TEMPERATURE_SCALE, hparams.f_attn_temp_scale, false); + ml.get_key(LLM_KV_ATTENTION_TEMPERATURE_LENGTH, hparams.attn_temp_length, false); + if (hparams.f_attn_temp_scale == 0.0f) { + throw std::runtime_error("beam: attention.temperature_scale must be nonzero"); + } + if (hparams.attn_temp_length == 0) { + throw std::runtime_error("beam: attention.temperature_length must be positive"); + } + hparams.n_attn_temp_floor_scale = hparams.attn_temp_length; + hparams.f_attn_temp_offset = 1.0f; + hparams.attn_temp_pow = true; + + switch (hparams.n_layer()) { + case 52: type = LLM_TYPE_UNKNOWN; break; + default: type = LLM_TYPE_UNKNOWN; + } +} + +void llama_model_beam::load_arch_tensors(llama_model_loader &) { + LLAMA_LOAD_LOCALS; + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + if (output == NULL) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, 0); + + create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0); + + layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0); + layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0); + + layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_embd_head_k * n_head}, 0); + layer.wqkv_gate_b = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "bias", i), {n_head}, 0); + + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, "weight", i), {n_embd}, 0); + + if ((uint32_t) i < hparams.n_layer_dense_lead) { + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + } else { + const int64_t n_ff_exp = hparams.n_ff_exp(i); + + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); + layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0); + + layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0); + layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0); + } + } +} + +std::unique_ptr llama_model_beam::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique(*this, params); +} + +static ggml_tensor * beam_residual_add(ggml_context * ctx, ggml_tensor * h, ggml_tensor * r, int i) { + const float h_scale = (float) (1.0 / std::sqrt((double) i)); + const float r_scale = (float) std::sqrt((double) (i - 1) / (double) i); + return ggml_add(ctx, ggml_scale(ctx, h, h_scale), ggml_scale(ctx, r, r_scale)); +} + +llama_model_beam::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v(); + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + ggml_tensor * inp_pos = build_inp_pos(); + + bool has_nope = false; + for (int il = 0; il < n_layer; ++il) { + has_nope = has_nope || !hparams.is_swa(il); + } + ggml_tensor * inp_attn_scale = has_nope ? build_inp_attn_scale() : nullptr; + + auto * inp_attn = build_attn_inp_kv_iswa(); + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + const float kq_scale = 1.0f / sqrtf(float(n_embd_head)); + + for (int il = 0; il < n_layer; ++il) { + const auto & layer = model.layers[il]; + + const bool is_swa = hparams.is_swa(il); + + ggml_tensor * inpSA = inpL; + + cur = build_norm(inpL, layer.attn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + { + ggml_tensor * attn_inp = cur; + + auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, n_embd_head, n_head, n_head_kv, il); + + Qcur = build_norm(Qcur, layer.attn_q_norm, NULL, LLM_NORM_RMS, il); + Kcur = build_norm(Kcur, layer.attn_k_norm, NULL, LLM_NORM_RMS, il); + cb(Qcur, "Qcur_normed", il); + cb(Kcur, "Kcur_normed", il); + + if (is_swa) { + const float freq_base_l = model.get_rope_freq_base (cparams, il); + const float freq_scale_l = model.get_rope_freq_scale(cparams, il); + + Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, + ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(Qcur, "Qcur_rope", il); + cb(Kcur, "Kcur_rope", il); + } else { + Qcur = ggml_mul(ctx0, Qcur, inp_attn_scale); + cb(Qcur, "Qcur_scaled", il); + } + + cur = build_attn(inp_attn, + NULL, NULL, NULL, // o_proj is applied after the gate + Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); + cb(cur, "attn_out", il); + + ggml_tensor * gate = build_lora_mm(layer.wqkv_gate, attn_inp); + cb(gate, "attn_gate_proj", il); + gate = ggml_reshape_3d(ctx0, gate, n_embd_head, n_head, n_tokens); + gate = ggml_add(ctx0, gate, ggml_reshape_3d(ctx0, layer.wqkv_gate_b, 1, n_head, 1)); + gate = ggml_sigmoid(ctx0, gate); + gate = ggml_reshape_2d(ctx0, gate, n_embd_head * n_head, n_tokens); + cb(gate, "attn_gate_sig", il); + + cur = ggml_mul(ctx0, cur, gate); + cb(cur, "attn_gated", il); + + cur = build_lora_mm(layer.wo, cur, layer.wo_s); + cb(cur, "attn_o_proj", il); + } + + cur = build_norm(cur, layer.attn_post_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "attn_post_norm", il); + + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + const int i_attn = 2 * (il + 1); + ggml_tensor * ffn_inp = beam_residual_add(ctx0, cur, inpSA, i_attn); + cb(ffn_inp, "ffn_inp", il); + + cur = build_norm(ffn_inp, layer.ffn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + if ((uint32_t) il < hparams.n_layer_dense_lead) { + cur = build_ffn(cur, + layer.ffn_up, NULL, NULL, + layer.ffn_gate, NULL, NULL, + layer.ffn_down, NULL, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + } else { + cur = build_moe_ffn(cur, + layer.ffn_gate_inp, + layer.ffn_up_exps, + layer.ffn_gate_exps, + layer.ffn_down_exps, + layer.ffn_exp_probs_b, + n_expert, n_expert_used, + LLM_FFN_SILU, + hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il); + cb(cur, "ffn_moe_out", il); + } + + cur = build_norm(cur, layer.ffn_post_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "ffn_post_norm", il); + + cur = beam_residual_add(ctx0, cur, ffn_inp, i_attn + 1); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1); + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur, model.output_s); + + if (hparams.f_final_logit_softcapping != 0.0f) { + cur = ggml_scale(ctx0, cur, 1.0f / hparams.f_final_logit_softcapping); + cur = ggml_tanh(ctx0, cur); + cur = ggml_scale(ctx0, cur, hparams.f_final_logit_softcapping); + } + + cb(cur, "result_output", -1); + res->t_logits = cur; +} diff --git a/src/models/models.h b/src/models/models.h index ffeed1733ac7..98f6b6e35dcc 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -2041,6 +2041,19 @@ struct llama_model_afmoe : public llama_model_base { }; +struct llama_model_beam : public llama_model_base { + llama_model_beam(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + struct graph : public llm_graph_context { + graph(const llama_model & model, const llm_graph_params & params); + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; + + struct llama_model_laguna : public llama_model_base { llama_model_laguna(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; diff --git a/tests/test-jinja.cpp b/tests/test-jinja.cpp index 7132c9e22483..1dacf170280e 100644 --- a/tests/test-jinja.cpp +++ b/tests/test-jinja.cpp @@ -2011,6 +2011,12 @@ static void test_object_methods(testing & t) { "a b " ); + test_template(t, "object | list", + "{{ (obj | list) == ['b', 'a'] }} {{ obj | list | join(',') }}", + {{"obj", {{"b", 1}, {"a", 2}}}}, + "True b,a" + ); + test_template(t, "object.values()", "{% for v in obj.values() %}{{ v }} {% endfor %}", {{"obj", {{"a", 1}, {"b", 2}}}}, diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index eb72553b3e81..14ca203be809 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -188,6 +188,11 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { } else if (arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_AFMOE || arch == LLM_ARCH_MAION_CODER) { n_head_kv = 2; // GQA coverage } + if (arch == LLM_ARCH_BEAM) { + n_head = 4; + n_head_kv = 2; + n_layer = 4; + } const uint32_t n_embd_head = n_embd / n_head; ms.add_kv(LLM_KV_GENERAL_ARCHITECTURE, llm_arch_name(arch)); @@ -294,6 +299,13 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { if (arch == LLM_ARCH_MAION_CODER) { ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, std::vector{1, 1, 0, 0}); + } else if (arch == LLM_ARCH_BEAM) { + std::vector pattern; + pattern.reserve(n_layer); + for (uint32_t il = 0; il < n_layer; il++) { + pattern.push_back(il % 2); + } + ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, pattern); } else if (arch == LLM_ARCH_GEMMA4) { ms.add_kv(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, n_embd/2); ms.add_kv(LLM_KV_ATTENTION_SHARED_KV_LAYERS, uint32_t(0)); @@ -470,6 +482,14 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { } } + if (arch == LLM_ARCH_BEAM) { + ms.add_kv(LLM_KV_EXPERT_COUNT, uint32_t(8)); + ms.add_kv(LLM_KV_EXPERT_GROUP_COUNT, uint32_t(4)); + ms.add_kv(LLM_KV_EXPERT_GROUP_USED_COUNT, uint32_t(2)); + ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 2.5f); + ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true); + } + if (arch == LLM_ARCH_MAION_CODER) { ms.add_kv(LLM_KV_MOE_LATENT_SIZE, n_embd / 2); ms.add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, std::vector{n_ff, n_ff / 2, n_ff, n_ff}); @@ -712,6 +732,7 @@ static bool moe_mandatory(const llm_arch arch) { case LLM_ARCH_BAILINGMOE3: case LLM_ARCH_DOTS1: case LLM_ARCH_AFMOE: + case LLM_ARCH_BEAM: case LLM_ARCH_ERNIE4_5: case LLM_ARCH_ERNIE4_5_MOE: case LLM_ARCH_HUNYUAN_MOE: From 2f2763db6eaac591e78023d1e2c4a31607c64517 Mon Sep 17 00:00:00 2001 From: Shawn Gu Date: Fri, 16 Oct 2026 15:15:04 -0700 Subject: [PATCH 188/431] opencl: Add MxFP4 MoE bin kernel for Adreno X1 devices (#30691) * opencl: add bin kernel `gemm_moe_mxfp4_q8_1_dp4a_ila_ldck` for X1 * opencl: fix whitespace * opencl: undo unnecessary change * opencl: enable dp4a clc for X1E --------- Co-authored-by: Li He --- ggml/src/ggml-opencl/CMakeLists.txt | 1 + ggml/src/ggml-opencl/ggml-opencl.cpp | 83 ++++++++++++++++++- .../kernels/transpose_activation_q8.cl | 35 ++++++++ 3 files changed, 115 insertions(+), 4 deletions(-) create mode 100644 ggml/src/ggml-opencl/kernels/transpose_activation_q8.cl diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt index 97b862afffd8..5ba63dade180 100644 --- a/ggml/src/ggml-opencl/CMakeLists.txt +++ b/ggml/src/ggml-opencl/CMakeLists.txt @@ -132,6 +132,7 @@ set(GGML_OPENCL_KERNELS gemm_moe_q6_k_q8_1_dp4a gemm_moe_q8_1_dp4a moe_reorder_quant_a_q8_1 + transpose_activation_q8 gemm_noshuffle_q4_k_q8_1_dp4a gemm_noshuffle_q5_k_q8_1_dp4a gemm_noshuffle_q6_k_q8_1_dp4a diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 17ceacc7aecc..de93b0134eb5 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -720,6 +720,10 @@ struct ggml_backend_opencl_context { ggml_cl_buffer prealloc_moe_qa; // int8 quants [tok_slots * ne00] ggml_cl_buffer prealloc_moe_da; // per-block d [tok_slots * ne00/32] (half) ggml_cl_buffer prealloc_moe_sa; // per-block s [tok_slots * ne00/32] (half) + // per-tile transposed qa/da and on-chip scratch for the ldck dp4a MoE GEMM + ggml_cl_buffer prealloc_moe_qa_trans; // int8 quants [tok_slots * ne00] + ggml_cl_buffer prealloc_moe_da_trans; // per-block d [tok_slots * ne00/32] (float) + ggml_cl_buffer prealloc_moe_qda_on_chip; // scratch copy of the router weights to avoid dst aliasing ggml_cl_buffer prealloc_moe_combine_w; ggml_cl_buffer prealloc_splitk_partial; // [ksplit * M] partials for split-K GEMV @@ -1016,6 +1020,7 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gemv_moe_q4_k_f32_ns_wimg = nullptr; // weight-as-texture MoE decode GEMV (opt-in) cl_kernel kernel_gemm_moe_q4_k_q8_1_dp4a = nullptr; // dp4a (int8) prefill GEMM variant cl_kernel kernel_moe_reorder_quant_a_q8_1; // fused reorder + q8_1 quant for the dp4a GEMM + cl_kernel kernel_transpose_activation_q8; // per-tile transpose of q8_1 activations for the ldck dp4a GEMM cl_kernel kernel_gemm_moe_q8_1_dp4a_q80 = nullptr; // generic dp4a MoE GEMM (MOE_QT=80), opt-in cl_kernel kernel_moe_expand_scale_q8_0 = nullptr; // q8_0 per-block d -> uniform scale[16] cl_kernel kernel_gemm_moe_q8_1_dp4a_q50 = nullptr; // generic dp4a MoE GEMM (MOE_QT=50, q5_0), opt-in @@ -1031,6 +1036,7 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a = nullptr; // dp4a (int8) mxfp4 MoE prefill GEMM cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a = nullptr; // dp4a (int8) q4_0 MoE prefill GEMM cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a_bin = nullptr; // binary dp4a (int8) mxfp4 MoE prefill GEMM + cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin = nullptr; // binary dp4a (int8) mxfp4 MoE prefill GEMM using ldck cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a_bin = nullptr; // binary dp4a (int8) q4_0 MoE prefill GEMM cl_kernel kernel_gemm_moe_q4_k_q8_1_dp4a_bin = nullptr; // binary dp4a (int8) q4_k MoE prefill GEMM cl_kernel kernel_gemm_moe_q6_k_q8_1_dp4a_bin = nullptr; // binary dp4a (int8) q6_k MoE prefill GEMM @@ -4903,6 +4909,24 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { } } + // gemm_moe_mxfp4_q8_1_dp4a_ldck_bin (dp4a prefill GEMM) + if (backend_ctx->has_integer_dot) { + size_t bin_size = 0; + backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin = nullptr; + + if (use_adreno_bin_kernels(backend_ctx)) { + const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_mxfp4_q8_1_dp4a_ila_ldck", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_q8_1_dp4a_ila_ldck", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } + } + // gemm_moe_q4_0_q8_1_dp4a (dp4a prefill GEMM) if (backend_ctx->has_integer_dot) { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -4984,6 +5008,23 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // transpose_activation_q8 (per-tile transpose of q8_1 activations for the ldck dp4a GEMM) + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "transpose_activation_q8.cl.h" + }; +#else + const std::string kernel_src = read_file("transpose_activation_q8.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts); + + CL_CHECK((backend_ctx->kernel_transpose_activation_q8 = clCreateKernel(prog, "kernel_transpose_activation_q8", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // gemv_moe_q5_k_f32_ns { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -28258,11 +28299,19 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, static const char * mxfp4_moe_dp4a_env = getenv("GGML_OPENCL_MXFP4_MOE_DP4A"); bool use_moe_dp4a = mxfp4_moe_dp4a_env ? (atoi(mxfp4_moe_dp4a_env) != 0) - : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E); // dot prod has to be available use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a; // bin kernel takes precedence, dp4a bin kernel has higher priority than normal bin kernel - if (backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin == nullptr) { + // ldck dp4a bin kernel (X1) has the same priority as normal bin kernel (X2/A8x), they are not expected to coexist. + const bool use_moe_dp4a_ldck = backend_ctx->has_integer_dot && + (!mxfp4_moe_dp4a_env || atoi(mxfp4_moe_dp4a_env) != 0) && + backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin == nullptr && + backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin == nullptr && + backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin != nullptr; + if (use_moe_dp4a_ldck) { + use_moe_dp4a = true; + } else if (backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin == nullptr) { use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin == nullptr; } @@ -28351,6 +28400,12 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char)); backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + if (use_moe_dp4a_ldck) { + backend_ctx->prealloc_moe_qa_trans.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char)); + backend_ctx->prealloc_moe_da_trans.allocate(backend_ctx->context, n_blocks * sizeof(cl_float)); + // matches max_constant_size of src1_a_cm in the ldck kernel + backend_ctx->prealloc_moe_qda_on_chip.allocate(backend_ctx->context, 2304); + } // fused reorder + q8_1 quant straight from the original // activations (no intermediate f32 reorder buffer). mxfp4 has no @@ -28375,13 +28430,33 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, cl_kernel dk = backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a; if (backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin) { dk = backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_bin; + } else if (use_moe_dp4a_ldck) { + dk = backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a_ldck_bin; + + // per-tile transpose of qa/da for the ldck kernel + cl_kernel tk = backend_ctx->kernel_transpose_activation_q8; + CL_CHECK(clSetKernelArg(tk, 0, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(tk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa_trans.buffer)); + CL_CHECK(clSetKernelArg(tk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(tk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da_trans.buffer)); + CL_CHECK(clSetKernelArg(tk, 4, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(tk, 5, sizeof(cl_uint), &ne00)); + size_t tk_local[3] = { 32, 8, 1 }; + size_t tk_global[3] = { 32, (size_t)(ne00 / 4), (size_t)max_post_router_tile }; + backend_ctx->enqueue_ndrange_kernel(tk, 3, tk_global, tk_local, dst); } int aidx = 0; CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_mxfp4->q_img)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_mxfp4->e)); - CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); - CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + if (use_moe_dp4a_ldck) { + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa_trans.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da_trans.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qda_on_chip.buffer)); + } else { + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + } CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap)); CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_dst_image)); diff --git a/ggml/src/ggml-opencl/kernels/transpose_activation_q8.cl b/ggml/src/ggml-opencl/kernels/transpose_activation_q8.cl new file mode 100644 index 000000000000..34d54b57e24a --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/transpose_activation_q8.cl @@ -0,0 +1,35 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable + +#define tile_size 32 + +kernel void kernel_transpose_activation_q8( + __global uint * src1_qa, + __global uint * src1_qa_trans, + __global half * src1_sa, + __global float * src1_sa_trans, + __global const int * total_tiles, + uint K +) { + uint row_in_tile = get_global_id(0); + uint blk_k = get_group_id(1); + uint col_in_tile = get_local_id(1); + uint tile_n = get_global_id(2); + + if (blk_k >= (K / tile_size) || tile_n >= (uint)total_tiles[0]) { + return; + } + + uint src_idx = (tile_n * tile_size + row_in_tile) * K / 4 + blk_k * 8 + col_in_tile; + uint dst_idx = tile_n * tile_size * K / 4 + (blk_k * 8 + col_in_tile) * tile_size + row_in_tile; + + src1_qa_trans[dst_idx] = src1_qa[src_idx]; + + if (col_in_tile == 0) { + src_idx = (tile_n * tile_size + row_in_tile) * K / 32 + blk_k; + dst_idx = tile_n * tile_size * K / 32 + blk_k * tile_size + row_in_tile; + + src1_sa_trans[dst_idx] = (float)(src1_sa[src_idx]); + } + +} From 27c7a769da1671c97b2e3f6cb88704a6e1e0d0cd Mon Sep 17 00:00:00 2001 From: wes Date: Sat, 17 Oct 2026 06:20:43 +0800 Subject: [PATCH 189/431] hexagon: vectorize softplus with HVX (#30435) softplus_f32 and tile_softplus_f32 ran a scalar logf(1.0f + expf(x)) loop. Replace both with a shared hvx_softplus_f32_aa() helper built from the backend's existing hvx_vec_exp_f32 / hvx_vec_log_f32 primitives, so no new numerical approximation is introduced. hvx_vec_exp_f32 clamps its input to (-88, 88) before exponentiating, so a naive log(1 + exp(x)) composition would saturate to ~88.72 for large x instead of returning x. The Q6_V_vmux_QVV select keeps the scalar reference's x > 20 guard (ggml_compute_softplus_f32 in ggml-impl.h), which is what makes the vector path match the CPU result. Assisted-by: Claude --- ggml/src/ggml-hexagon/htp/unary-ops.c | 51 +++++++++++++++++++-------- 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.c index c557b0907daa..37df31ddba81 100644 --- a/ggml/src/ggml-hexagon/htp/unary-ops.c +++ b/ggml/src/ggml-hexagon/htp/unary-ops.c @@ -690,22 +690,48 @@ static void tri_f32(const void * restrict src, } } +// softplus(x) = log(1 + exp(x)) +// Match CPU reference: return x for x > 20 to avoid overflow. +static inline void hvx_softplus_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + const HVX_Vector * restrict v_src = (const HVX_Vector *) src; + HVX_Vector * restrict v_dst = (HVX_Vector *) dst; + + const HVX_Vector twenty = hvx_vec_splat_f32(20.0f); + const HVX_Vector one = hvx_vec_splat_f32(1.0f); + + const uint32_t nvec = n / VLEN_FP32; + const uint32_t nloe = n % VLEN_FP32; + + #pragma unroll(4) + for (uint32_t i = 0; i < nvec; i++) { + HVX_Vector x = v_src[i]; + HVX_VectorPred pred_gt20 = Q6_Q_vcmp_gt_VsfVsf(x, twenty); + HVX_Vector expx = hvx_vec_exp_f32(x); + HVX_Vector logv = hvx_vec_log_f32(hvx_vec_add_f32_f32(one, expx)); + v_dst[i] = Q6_V_vmux_QVV(pred_gt20, x, logv); + } + + if (nloe > 0) { + HVX_Vector x = v_src[nvec]; + HVX_VectorPred pred_gt20 = Q6_Q_vcmp_gt_VsfVsf(x, twenty); + HVX_Vector expx = hvx_vec_exp_f32(x); + HVX_Vector logv = hvx_vec_log_f32(hvx_vec_add_f32_f32(one, expx)); + HVX_Vector sp = Q6_V_vmux_QVV(pred_gt20, x, logv); + hvx_vec_store_a((void *) &v_dst[nvec], nloe * sizeof(float), sp); + } +} + static void softplus_f32(const void * restrict src, void * restrict dst, const uint32_t num_rows, const struct htp_unary_context * uctx) { htp_unary_op_preamble; - // softplus(x) = log(1 + exp(x)) - // Match CPU reference: ggml_compute_softplus_f32() in ggml-impl.h + for (uint32_t ir = 0; ir < num_rows; ir++) { - const float * restrict src_f = (const float *)((const uint8_t *)src + (ir * src0_row_size_aligned)); - float * restrict dst_f = (float *)((uint8_t *)dst + (ir * dst_row_size_aligned)); + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); - for (uint32_t i = 0; i < ne0; i++) { - float x = src_f[i]; - // For x > 20: softplus(x) ~ x (avoids exp overflow) - dst_f[i] = (x > 20.0f) ? x : logf(1.0f + expf(x)); - } + hvx_softplus_f32_aa(dst_local, src_local, ne0); } } @@ -866,12 +892,7 @@ static void tile_gelu_erf_f32(void * restrict dst, const void * restrict src, ui static void tile_softplus_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { (void) uctx; - const float * restrict sf = (const float *) src; - float * restrict df = (float *) dst; - for (uint32_t i = 0; i < tw; i++) { - float x = sf[i]; - df[i] = (x > 20.0f) ? x : logf(1.0f + expf(x)); - } + hvx_softplus_f32_aa((uint8_t *) dst, (const uint8_t *) src, tw); } static void tile_tanh_f32(void * restrict dst, const void * restrict src, uint32_t tw, const struct htp_unary_context * uctx) { From 6420cf5dd588c68e48657a0bbd82bb3bd5ac7209 Mon Sep 17 00:00:00 2001 From: Zack Li <39573601+zhiyuan8@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:25:05 -0700 Subject: [PATCH 190/431] ci: update device matrix for Snapdragon jobs by removing SM8750 (#30620) * ci: update device matrix for Snapdragon jobs by removing SM8750 * ci: increase timeout for QDC device tests and update job timeout parameters --- .github/workflows/build-and-test-snapdragon.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-test-snapdragon.yml b/.github/workflows/build-and-test-snapdragon.yml index d6420b65f141..29be95c8be02 100644 --- a/.github/workflows/build-and-test-snapdragon.yml +++ b/.github/workflows/build-and-test-snapdragon.yml @@ -104,11 +104,11 @@ jobs: name: Test on QDC Device (${{ matrix.device }}) needs: [android-ndk-snapdragon, linux-iot-snapdragon] runs-on: ubuntu-24.04-arm - timeout-minutes: 90 + timeout-minutes: 180 strategy: fail-fast: false matrix: - device: [SM8750, SM8850, QCS9075M] + device: [SM8850, QCS9075M] permissions: actions: read @@ -155,7 +155,7 @@ jobs: --pkg-dir pkg-snapdragon/llama.cpp \ --model-url "https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_0.gguf" \ --device ${{ matrix.device }} \ - ${{ startsWith(matrix.device, 'QCS') && '--retries 2 --retry-delay 300' || '' }} + ${{ startsWith(matrix.device, 'QCS') && '--job-timeout 5400 --retries 2 --retry-delay 300' || '' }} env: QDC_API_KEY: ${{ secrets.QDC_API_KEY }} From ea0c4c2c85fb183ec0fbd30a1af4e4275a18a064 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Sat, 17 Oct 2026 02:11:56 +0200 Subject: [PATCH 191/431] skills: add slice-model (#30677) * skills: add slice-model * exclude from py check * add to agents.md --- .flake8 | 2 + AGENTS.md | 3 + skills/slice-model/SKILL.md | 26 ++ skills/slice-model/slice_model.py | 558 ++++++++++++++++++++++++++++++ ty.toml | 1 + 5 files changed, 590 insertions(+) create mode 100644 skills/slice-model/SKILL.md create mode 100644 skills/slice-model/slice_model.py diff --git a/.flake8 b/.flake8 index 669d231f1f63..c7b5cf7f803d 100644 --- a/.flake8 +++ b/.flake8 @@ -5,6 +5,8 @@ exclude = # Do not traverse examples and tools examples, tools, + # Reference scripts for skills + skills, # Do not include package initializers __init__.py, # No need to traverse our git directory diff --git a/AGENTS.md b/AGENTS.md index 3ae583b82bae..36390a6b4ad6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -246,3 +246,6 @@ Chat template and parser: - [PEG parser](docs/development/parsing.md) - alternative to regex that llama.cpp uses to parse model's output - [Auto parser](docs/autoparser.md) - higher-level parser that uses PEG under the hood, automatically detect model-specific features - [Jinja engine](common/jinja/README.md) + +Testing: +- [Slice model](skills/slice-model/SKILL.md) - build a small slice (few layers/experts) of a big HF model without downloading the full weights, to test conversion and inference diff --git a/skills/slice-model/SKILL.md b/skills/slice-model/SKILL.md new file mode 100644 index 000000000000..f19564f90bd3 --- /dev/null +++ b/skills/slice-model/SKILL.md @@ -0,0 +1,26 @@ +--- +name: slice-model +description: Build a small test "slice" of a HF model (fewer layers/experts) by downloading only the needed tensors via HTTP range requests. Use when the user needs a small version of a big model to test conversion or inference. +--- + +# Slice a HF model for testing + +[slice_model.py](slice_model.py) is a **reference** implementation. Read it first: its docstring and `--help` explain how it works and what it does. Its heuristics only cover common layouts, and new models often have tensor names, shapes or config keys it does not handle. Do not run it blindly. If the model needs special handling, copy it to `tmp/slice_.py` and adapt the copy, or write a new script with the same approach. + +## Steps + +1. Ask the user for the repo, what the slice must exercise (e.g. a MoE layer, each layer type of a hybrid model, MTP, the vision encoder), the size budget and the output dir. +2. Inspect the model: run the script with `--inspect`, read `config.json`, and read the HF modeling code if anything is unclear. Check the plan against the pitfalls below. +3. Build the slice, then read the config patches in the log and check each one. +4. Verify: load the slice in transformers (no missing/unexpected weight warnings, no NaN logits), then convert it with `convert_hf_to_gguf.py` and run it. For numerical checks, see `examples/model-conversion/`. + +## Pitfalls + +- Layer counts in nested or unknown config keys, or several towers with the same layer count. +- Hybrid layer patterns (`layer_types`, `full_attention_interval`, ...): keep at least one layer of each type, and check the patched config still matches the kept layers. +- Dense leading layers (`first_k_dense_replace`) and MTP/nextn layers numbered after the regular layers. +- Every per-expert tensor must be sliced to the same expert count: experts, router weight and biases. Tensors with the expert dim not at dim 0 are not contiguous. +- Routing constraints: top-k, expert groups, shared experts, zero-compute experts. +- Quantized weights (FP8, MXFP4): slice the scale tensors in the same way as their weight. +- Sizes derived in modeling code instead of stored in the config: follow the exact formula if you shrink them. +- Slicing the vocab without the tokenizer breaks conversion. diff --git a/skills/slice-model/slice_model.py b/skills/slice-model/slice_model.py new file mode 100644 index 000000000000..88d696a5eda9 --- /dev/null +++ b/skills/slice-model/slice_model.py @@ -0,0 +1,558 @@ +#!/usr/bin/env python3 +""" +REFERENCE ONLY - build a small test "slice" of a HF safetensors model without downloading the full weights. + +The heuristics below (layer/expert detection, config patching) cover common layouts only. New models often use tensor names, shapes or config keys that this script does not know about. Always run --inspect first, check the plan, and adapt the script to the model. + +How it works: each safetensors shard starts with an 8-byte header length + JSON header, which gives the exact byte span of every tensor. We fetch only the headers, then use HTTP range requests to fetch the tensors we keep: + 1. keep only some layers (per numeric "family" in tensor names whose count matches a layer count in config.json), renumbered to 0..n-1 + 2. keep only the first E experts (per-expert tensors are dropped, stacked expert tensors and router weights are row-sliced along dim 0) + 3. optionally keep only the first V rows of vocab-sized tensors (off by default, see --vocab-size) +config.json is patched to match. Other small repo files are copied as-is. + +Usage: + python slice_model.py Qwen/Qwen3-0.6B --inspect + python slice_model.py Qwen/Qwen3-30B-A3B --num-layers 2 --num-experts 8 --dry-run + python slice_model.py Qwen/Qwen3-30B-A3B --keep-layers 0,3 --output ../models/qwen3moe-slice +""" +from __future__ import annotations + +import argparse +import concurrent.futures as cf +import fnmatch +import json +import logging +import os +import re +import struct +import sys +import threading +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import requests +from huggingface_hub import HfApi, get_token, hf_hub_url + +logging.basicConfig(level=logging.INFO, format="%(message)s") +logger = logging.getLogger("slice_model") + +INDEX_FILE = "model.safetensors.index.json" +CONFIG_FILE = "config.json" + +# files not worth copying into a slice +AUX_BLACKLIST = [ + # weights in other formats (the slice replaces them) + "*.bin", "*.pth", "*.pt", "*.ckpt", "*.h5", "*.msgpack", + "*.onnx", "*.tflite", "*.mlmodel", "*.mlpackage/*", "*.gguf", "*.nemo", + "onnx/*", "openvino/*", "coreml/*", "original/*", "metal/*", + # media / demo assets + "*.png", "*.jpg", "*.jpeg", "*.gif", "*.webp", "*.avif", "*.svg", + "*.mp4", "*.webm", "*.mov", "*.wav", "*.mp3", "*.flac", "*.ogg", "*.pdf", + ".gitattributes", +] +AUX_MAX_SIZE = 50 * 1024 * 1024 + +LAYER_CONTAINERS = {"layers", "layer", "blocks", "block", "h"} +EXPERT_ROW_KEYWORDS = ("expert", "router", "gate", "score") + +LAYER_KEYS = { + "num_hidden_layers", "num_layers", "n_layer", "n_layers", "num_layer", + "num_decoder_layers", "encoder_layers", "decoder_layers", "depth", + "num_encoder_layers", "num_attention_layers", "layer_count", +} +VOCAB_KEYS = {"vocab_size", "padded_vocab_size"} +NEXTN_KEYS = {"num_nextn_predict_layers", "num_mtp_layers"} +TOPK_SUBSTR = ("per_tok", "topk", "top_k", "moe_k") +EXPERT_KEY_EXCLUDE = ("per_tok", "topk", "top_k", "shared", "group", "intermediate", "size", "dim", "dtype") + + +def parse_size(text: str) -> int: + m = re.fullmatch(r"(\d+(?:\.\d+)?)\s*([kKmMgGtT]?)[bB]?", text.strip()) + if not m: + raise argparse.ArgumentTypeError(f"invalid size: {text!r}") + mult = {"": 1, "k": 1024, "m": 1024**2, "g": 1024**3, "t": 1024**4}[m.group(2).lower()] + return int(float(m.group(1)) * mult) + + +def human(n: float) -> str: + for unit in ("B", "KB", "MB", "GB"): + if n < 1024 or unit == "GB": + return f"{n:.1f}{unit}" if unit != "B" else f"{int(n)}B" + n /= 1024 + return f"{n}GB" + + +# -------------------------------------------------------------------------------------- +# HTTP +# -------------------------------------------------------------------------------------- + +_tls = threading.local() + + +def _session() -> requests.Session: + s = getattr(_tls, "s", None) + if s is None: + s = requests.Session() + _tls.s = s + return s + + +class HubClient: + def __init__(self, repo_id: str, revision: str, token: str | None): + self.repo_id = repo_id + self.revision = revision + self.headers = {"Authorization": f"Bearer {token}"} if token else {} + + def url(self, filename: str) -> str: + return hf_hub_url(self.repo_id, filename, revision=self.revision) + + def _request(self, filename: str, headers: dict, stream: bool = False, retries: int = 5): + last: Exception | None = None + for attempt in range(1, retries + 1): + try: + r = _session().get(self.url(filename), headers=headers, stream=stream, timeout=90, allow_redirects=True) + if r.status_code in (429, 500, 502, 503, 504): + raise requests.HTTPError(f"HTTP {r.status_code}", response=r) + r.raise_for_status() + return r + except Exception as e: # noqa: BLE001 + last = e + if attempt == retries or "404" in str(e) or "401" in str(e) or "403" in str(e): + break + time.sleep(2.0 * attempt) + raise RuntimeError(f"{self.repo_id}/{filename}: {last}") from last + + def get_range(self, filename: str, start: int, end_inclusive: int) -> bytes: + h = dict(self.headers) + h["Range"] = f"bytes={start}-{end_inclusive}" + return self._request(filename, h).content + + def get_json(self, filename: str) -> Any: + return json.loads(self._request(filename, dict(self.headers)).content) + + def download_file(self, filename: str, dest: Path): + r = self._request(filename, dict(self.headers), stream=True) + dest.parent.mkdir(parents=True, exist_ok=True) + with open(dest, "wb") as f: + for chunk in r.iter_content(chunk_size=4 * 1024 * 1024): + if chunk: + f.write(chunk) + + def stream_range_into_fd(self, filename: str, start: int, end_inclusive: int, fd: int, write_offset: int): + h = dict(self.headers) + h["Range"] = f"bytes={start}-{end_inclusive}" + r = self._request(filename, h, stream=True) + pos = write_offset + for chunk in r.iter_content(chunk_size=8 * 1024 * 1024): + if chunk: + os.pwrite(fd, chunk, pos) + pos += len(chunk) + if pos - write_offset != end_inclusive - start + 1: + raise RuntimeError(f"short read: {filename} bytes={start}-{end_inclusive}") + + +def fetch_shard_header(client: HubClient, filename: str) -> tuple[dict, int]: + (header_len,) = struct.unpack(" ConfigInfo: + info = ConfigInfo(cfg=cfg) + if cfg is None: + return info + for _d, key, val in walk_config(cfg): + if not isinstance(val, int) or isinstance(val, bool) or val <= 0: + continue + if key in LAYER_KEYS: + info.layer_counts.add(val) + elif key in NEXTN_KEYS: + info.nextn_counts.add(val) + elif key in VOCAB_KEYS: + info.vocab_sizes.add(val) + elif key == "first_k_dense_replace": + info.first_k_dense = val + elif "expert" in key and val > 1 and not any(x in key for x in EXPERT_KEY_EXCLUDE): + info.expert_counts.add(val) + return info + + +def patch_config(cfg: dict, families: dict[str, Family], new_vocab: int | None, orig_vocabs: set[int]) -> list[str]: + """Patch counts in-place to match the slice. Returns log lines.""" + log = [] + by_card: dict[int, Family] = {} + for fam in families.values(): + if fam.kept is not None: + by_card.setdefault(fam.card, fam) + by_card.setdefault(fam.regular_total, fam) + max_new_experts = max((f.new_count for f in families.values() if f.kind == "expert" and f.kept is not None), default=None) + + for d, key, val in list(walk_config(cfg)): + if isinstance(val, int) and not isinstance(val, bool): + fam = by_card.get(val) + if key in LAYER_KEYS and fam is not None and fam.kind == "layer": + d[key] = fam.new_regular + elif key == "first_k_dense_replace": + lfam = next((f for f in families.values() if f.kind == "layer"), None) + if lfam is None: + continue + d[key] = sum(1 for i in lfam.kept if i < val) + elif "expert" in key and fam is not None and fam.kind == "expert" and not any(x in key for x in EXPERT_KEY_EXCLUDE): + d[key] = fam.new_count + elif "expert" in key and any(x in key for x in TOPK_SUBSTR) and max_new_experts is not None and val > max_new_experts: + d[key] = max_new_experts + elif key in VOCAB_KEYS and new_vocab is not None and val in orig_vocabs: + d[key] = new_vocab + if d[key] != val: + log.append(f"{key}: {val} -> {d[key]}") + elif isinstance(val, list) and val and all(not isinstance(x, (dict, list)) for x in val): + # per-layer lists, e.g. layer_types + fam = by_card.get(len(val)) + if fam is not None and fam.kind == "layer" and len(val) in (fam.card, fam.regular_total): + d[key] = [val[i] for i in fam.kept if i < len(val)] + log.append(f"{key}: list[{len(val)}] -> list[{len(d[key])}]") + return log + + +# -------------------------------------------------------------------------------------- +# tensor name analysis +# -------------------------------------------------------------------------------------- + +@dataclass +class Family: + """One numeric slot in tensor names, e.g. 'model.layers.#'.""" + prefix: str + kind: str # "layer" | "expert" | "other" + indices: set[int] = field(default_factory=set) + kept: list[int] | None = None # None = keep all, no renumbering + renumber: dict[int, int] = field(default_factory=dict) + regular_total: int = 0 # layer count excluding MTP tail + new_regular: int = 0 + + @property + def card(self) -> int: + return len(self.indices) + + @property + def new_count(self) -> int: + return len(self.kept) if self.kept is not None else self.card + + +def name_slots(name: str): + """Yield (slot_pos, family_prefix, index) for each numeric path component.""" + tokens = name.split(".") + norm: list[str] = [] + for i, tok in enumerate(tokens): + if tok.isdigit(): + yield i, ".".join(norm), int(tok) + norm.append("#") + else: + norm.append(tok) + + +def build_families(names: list[str], cfg_info: ConfigInfo) -> dict[str, Family]: + families: dict[str, Family] = {} + for name in names: + tokens = name.split(".") + for i, prefix, idx in name_slots(name): + container = tokens[i - 1] if i > 0 else "" + fam = families.get(prefix) + if fam is None: + fam = Family(prefix=prefix, kind="expert" if "expert" in container else "other") + families[prefix] = fam + fam.indices.add(idx) + + acceptable_layer_cards: dict[int, int] = {} # card -> regular layer count + for c in cfg_info.layer_counts: + acceptable_layer_cards[c] = c + for m in cfg_info.nextn_counts: + acceptable_layer_cards[c + m] = c + + for fam in families.values(): + if fam.kind == "expert": + continue + container = fam.prefix.rsplit(".", 1)[-1] + if cfg_info.cfg is not None: + if fam.card in acceptable_layer_cards: + fam.kind = "layer" + fam.regular_total = acceptable_layer_cards[fam.card] + elif fam.card >= 4 and container in LAYER_CONTAINERS: + fam.kind = "layer" + fam.regular_total = fam.card + return families + + +def plan_families(families: dict[str, Family], num_layers: int, keep_layers: list[int] | None, num_experts: int, first_k_dense: int): + for fam in families.values(): + if fam.kind == "layer": + if keep_layers is not None: + kept = [i for i in keep_layers if i < fam.regular_total] + else: + base = min(num_layers, fam.regular_total) + kept = list(range(base)) + # keep one MoE layer for models whose first k layers are dense + if 0 < first_k_dense < fam.regular_total and first_k_dense >= base >= 1: + kept = list(range(base - 1)) + [first_k_dense] + fam.new_regular = len(kept) + # keep the MTP tail (layer indices beyond the regular count) + kept += [i for i in sorted(fam.indices) if i >= fam.regular_total] + fam.kept = sorted(set(kept)) + elif fam.kind == "expert": + fam.kept = sorted(fam.indices)[:num_experts] + fam.new_regular = len(fam.kept) + else: + fam.kept = None + continue + fam.renumber = {old: new for new, old in enumerate(fam.kept)} + + +def slice_tensor_name(name: str, families: dict[str, Family]) -> str | None: + """Return the renumbered output name, or None if the tensor is dropped.""" + tokens = name.split(".") + for i, prefix, idx in name_slots(name): + fam = families[prefix] + if fam.kept is None: + continue + if idx not in fam.renumber: + return None + tokens[i] = str(fam.renumber[idx]) + return ".".join(tokens) + + +# -------------------------------------------------------------------------------------- +# slicing +# -------------------------------------------------------------------------------------- + +@dataclass +class TensorPlan: + src_name: str + out_name: str + dtype: str + shape: list[int] + orig_shape: list[int] + size: int + src_file: str + seg_start: int # absolute byte range in src_file + seg_end: int # inclusive + + +def pick_weight_files(filenames: list[str]) -> tuple[list[str], bool]: + """Return (weight files, has_index). Preference: index -> model.safetensors -> any root safetensors.""" + if INDEX_FILE in filenames: + return [], True + if "model.safetensors" in filenames: + return ["model.safetensors"], False + root_st = [f for f in filenames if f.endswith(".safetensors") and "/" not in f] + if "consolidated.safetensors" in root_st: + return ["consolidated.safetensors"], False + return root_st[:1], False + + +def row_slice(shape: list[int], size: int, keep_rows: int) -> tuple[list[int], int] | None: + # only a prefix of dim 0 is a single contiguous byte range + if len(shape) < 1 or shape[0] <= keep_rows or size % shape[0] != 0: + return None + row_bytes = size // shape[0] + return [keep_rows] + shape[1:], keep_rows * row_bytes + + +def build_plan(args, families, headers, weight_files, src_file_of, cfg_info) -> list[TensorPlan]: + plan_families(families, args.num_layers, args.keep_layers, args.num_experts, cfg_info.first_k_dense) + plans = [] + for fname in weight_files: + header, data_start = headers[fname] + for name, meta in header.items(): + out_name = slice_tensor_name(name, families) + if out_name is None: + continue + start, end = meta["data_offsets"] + shape, size = list(meta["shape"]), end - start + new_rows = None + if args.vocab_size and shape and shape[0] > args.vocab_size \ + and any(shape[0] >= vs and shape[0] - vs <= 1024 for vs in cfg_info.vocab_sizes): + new_rows = args.vocab_size + elif shape and shape[0] in cfg_info.expert_counts and shape[0] > args.num_experts \ + and any(k in name for k in EXPERT_ROW_KEYWORDS): + new_rows = args.num_experts + if new_rows is not None: + sliced = row_slice(shape, size, new_rows) + if sliced is not None: + shape, size = sliced + plans.append(TensorPlan(name, out_name, meta["dtype"], shape, list(meta["shape"]), size, src_file_of[name], + data_start + start, data_start + start + size - 1)) + return plans + + +def print_inspect(cfg_info: ConfigInfo, families: dict[str, Family], plans: list[TensorPlan], all_names: list[str]): + logger.info("=== config analysis ===") + logger.info(f"layer counts: {sorted(cfg_info.layer_counts)} nextn: {sorted(cfg_info.nextn_counts)} " + f"experts: {sorted(cfg_info.expert_counts)} vocab: {sorted(cfg_info.vocab_sizes)} first_k_dense: {cfg_info.first_k_dense}") + logger.info("\n=== numeric families in tensor names ===") + for fam in sorted(families.values(), key=lambda f: f.prefix): + kept = "all" if fam.kept is None else f"{fam.kept[:16]}{'...' if len(fam.kept) > 16 else ''}" + logger.info(f"{fam.prefix:60s} kind={fam.kind:6s} count={fam.card:4d} kept={kept}") + logger.info("\n=== kept tensors (first index of each family only) ===") + seen = set() + for t in plans: + key = re.sub(r"\.\d+\.", ".#.", t.out_name) + if key in seen and t.shape == t.orig_shape: + continue + seen.add(key) + mark = f" (sliced from {t.orig_shape})" if t.shape != t.orig_shape else "" + logger.info(f"{t.out_name:70s} {t.dtype:5s} {t.shape}{mark}") + kept_src = {t.src_name for t in plans} + dropped = [n for n in all_names if n not in kept_src] + logger.info(f"\n{len(plans)}/{len(all_names)} tensors kept, {len(dropped)} dropped") + + +def write_slice(out_dir: Path, client: HubClient, plans: list[TensorPlan], jobs: int): + header_out: dict[str, Any] = {} + offset = 0 + for t in plans: + header_out[t.out_name] = {"dtype": t.dtype, "shape": t.shape, "data_offsets": [offset, offset + t.size]} + offset += t.size + header_out["__metadata__"] = {"format": "pt"} + hbytes = json.dumps(header_out, separators=(",", ":")).encode() + hbytes += b" " * ((-(len(hbytes) + 8)) % 8) + data_start = 8 + len(hbytes) + + shard_path = out_dir / "model.safetensors" + with open(shard_path, "wb") as f: + f.write(struct.pack(" args.max_size: + logger.error(f"slice is {human(est)} > --max-size {human(args.max_size)}") + return 1 + if args.inspect or args.dry_run: + return 0 + + out_dir = args.output or Path("tmp") / "hf_slices" / args.repo.replace("/", "--") + out_dir.mkdir(parents=True, exist_ok=True) + write_slice(out_dir, client, plans, args.jobs) + + skipped_aux = [] + for fname in filenames: + if fname in weight_files or fname in (INDEX_FILE, CONFIG_FILE) or fname.endswith(".safetensors"): + continue + if any(fnmatch.fnmatch(fname, p) or fnmatch.fnmatch(Path(fname).name, p) for p in AUX_BLACKLIST): + continue + if siblings[fname] > AUX_MAX_SIZE: + skipped_aux.append(f"{fname} ({human(siblings[fname])})") + continue + client.download_file(fname, out_dir / fname) + for f in skipped_aux: + logger.warning(f"skipped large aux file {f}") + + patch_log = [] + if cfg is not None: + new_vocab = args.vocab_size if args.vocab_size and any(vs > args.vocab_size for vs in cfg_info.vocab_sizes) else None + patch_log = patch_config(cfg, families, new_vocab, cfg_info.vocab_sizes) + with open(out_dir / CONFIG_FILE, "w") as f: + json.dump(cfg, f, indent=2) + f.write("\n") + for line in patch_log: + logger.info(f"config: {line}") + + with open(out_dir / ".slice_meta.json", "w") as f: + json.dump({ + "repo": args.repo, "revision": info.sha, + "tensors": len(plans), "total_tensors": len(all_names), "size": est, + "config_patches": patch_log, "skipped_aux": skipped_aux, + }, f, indent=2) + logger.info(f"wrote {out_dir}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ty.toml b/ty.toml index 340b0649d334..e2036cd2ed05 100644 --- a/ty.toml +++ b/ty.toml @@ -8,6 +8,7 @@ deprecated = "warn" [src] exclude = [ "./tools/mtmd/legacy-models/**", + "./skills/**", ] [[overrides]] From f74e2fa6a31cc33ab4ee4ae516641cfe62df3924 Mon Sep 17 00:00:00 2001 From: Marcus Date: Fri, 16 Oct 2026 19:03:41 -0700 Subject: [PATCH 192/431] sycl: fix prompt-processing regression after Q4_0/Q8_0 weight reorder (#30374) --- ggml/src/ggml-sycl/convert.cpp | 30 ++++++------- ggml/src/ggml-sycl/dequantize.hpp | 74 ++++++++++++------------------- 2 files changed, 43 insertions(+), 61 deletions(-) diff --git a/ggml/src/ggml-sycl/convert.cpp b/ggml/src/ggml-sycl/convert.cpp index 67dfeb9f943e..8ba26d3a9f23 100644 --- a/ggml/src/ggml-sycl/convert.cpp +++ b/ggml/src/ggml-sycl/convert.cpp @@ -158,16 +158,16 @@ static void dequantize_row_q4_0_sycl_reorder(const void *vx, dst_t *y, const int dpct::has_capability_or_fail(stream->get_device(), {sycl::aspect::fp16}); - int constexpr WARP_K = WARP_SIZE * QK4_0; - const int n_warp = (k + WARP_K - 1) / WARP_K; - GGML_ASSERT(k % 2 == 0); - stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, n_warp) * - sycl::range<3>(1, 1, WARP_SIZE), - sycl::range<3>(1, 1, WARP_SIZE)), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + GGML_ASSERT(k % QK4_0 == 0); + const int64_t n_groups = ceil_div(k / 2, SYCL_DEQUANTIZE_BLOCK_SIZE); + // decrease global range when it exceeds the max int; the kernel loops to cover the rest + const int64_t wg_size = downsample_sycl_global_range(n_groups, SYCL_DEQUANTIZE_BLOCK_SIZE); + stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, n_groups) * + sycl::range<3>(1, 1, wg_size), + sycl::range<3>(1, 1, wg_size)), + [=](sycl::nd_item<3> item_ct1) { dequantize_block_q4_0_reorder(vx, y, k, item_ct1); }); - } template @@ -177,16 +177,16 @@ static void dequantize_row_q8_0_sycl_reorder(const void *vx, dst_t *y, const int dpct::has_capability_or_fail(stream->get_device(), {sycl::aspect::fp16}); - int constexpr WARP_K = WARP_SIZE * QK8_0; - const int n_warp = (k + WARP_K - 1) / WARP_K; GGML_ASSERT(k % QK8_0 == 0); - stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, n_warp) * - sycl::range<3>(1, 1, WARP_SIZE), - sycl::range<3>(1, 1, WARP_SIZE)), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + const int64_t n_groups = ceil_div(k / 2, SYCL_DEQUANTIZE_BLOCK_SIZE); + // decrease global range when it exceeds the max int; the kernel loops to cover the rest + const int64_t wg_size = downsample_sycl_global_range(n_groups, SYCL_DEQUANTIZE_BLOCK_SIZE); + stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, n_groups) * + sycl::range<3>(1, 1, wg_size), + sycl::range<3>(1, 1, wg_size)), + [=](sycl::nd_item<3> item_ct1) { dequantize_block_q8_0_reorder(vx, y, k, item_ct1); }); - } template diff --git a/ggml/src/ggml-sycl/dequantize.hpp b/ggml/src/ggml-sycl/dequantize.hpp index 14529a26beb3..48ee651ad68c 100644 --- a/ggml/src/ggml-sycl/dequantize.hpp +++ b/ggml/src/ggml-sycl/dequantize.hpp @@ -818,61 +818,43 @@ static void dequantize_block_q4_0(const void * __restrict__ vx, dst_t * __restri } template -static void dequantize_block_q4_0_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t nb32, +static void dequantize_block_q4_0_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t k, const sycl::nd_item<3> &item_ct1) { - - const int64_t i = item_ct1.get_group(2); - auto k=nb32; - // assume 32 threads - const int64_t tid = item_ct1.get_local_id(2); - const int lane_ib = i * WARP_SIZE + tid; - - if (lane_ib >= k / QK4_0) { - return; + // one thread per qs byte, like dequantize_block; the reorder layout makes the byte reads contiguous + // grid-stride loop because the launcher may shrink the work-group to keep the global range below INT_MAX + // float math, not dequantize_q4_0_reorder: that helper computes in half with GGML_SYCL_F16, which changes f32 output + const sycl::half * d_ptr = (const sycl::half *) ((const uint8_t *) vx + k/2); + const int64_t stride = item_ct1.get_local_range(2) * item_ct1.get_group_range(2); + + for (int64_t iq = item_ct1.get_global_id(2); iq < k/2; iq += stride) { + const int64_t ib = iq / (QK4_0/2); + const int64_t iqs = iq % (QK4_0/2); + const int vq = ((const uint8_t *) vx)[iq]; + const float d = float(d_ptr[ib]); + + dst_t * y = yy + ib*QK4_0 + iqs; + y[0] = d * ((vq & 0xF) - 8); + y[QK4_0/2] = d * ((vq >> 4) - 8); } - - dst_t * y_ptr = yy + lane_ib * QK4_0; - - auto qs = (const uint8_t*)vx + lane_ib * QK4_0 / 2; - auto s_ptr = (const sycl::half*)((const uint8_t*)vx + k / 2) + lane_ib; - - const float d = float(*s_ptr); - -#pragma unroll - for (int l = 0; l < QK4_0 / 2; ++l) { - int vq = qs[l]; - y_ptr[l + 0] = d * ((vq & 0xF) - 8); - y_ptr[l + 16] = d * ((vq >> 4) - 8); - } - } // Dequantize Q8_0 from reorder layout: [all qs (k bytes)][all d values] -// Each thread handles one block of QK8_0 elements. template static void dequantize_block_q8_0_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t k, const sycl::nd_item<3> &item_ct1) { - - const int64_t i = item_ct1.get_group(2); - const int64_t tid = item_ct1.get_local_id(2); - const int lane_ib = i * WARP_SIZE + tid; - - if (lane_ib >= k / QK8_0) { - return; - } - - dst_t * y_ptr = yy + lane_ib * QK8_0; - - auto qs = (const int8_t*)vx + lane_ib * QK8_0; - auto s_ptr = (const sycl::half*)((const uint8_t*)vx + k) + lane_ib; - - const float d = float(*s_ptr); - -#pragma unroll - for (int l = 0; l < QK8_0; ++l) { - y_ptr[l] = d * qs[l]; + // two values per thread, like dequantize_block; the reorder layout keeps all qs contiguous + // grid-stride loop because the launcher may shrink the work-group to keep the global range below INT_MAX + // float math, not dequantize_q8_0_reorder: that helper computes in half with GGML_SYCL_F16, which changes f32 output + const int8_t * qs = (const int8_t *) vx; + const sycl::half * d_ptr = (const sycl::half *) ((const uint8_t *) vx + k); + const int64_t stride = 2 * item_ct1.get_local_range(2) * item_ct1.get_group_range(2); + + for (int64_t i = 2 * item_ct1.get_global_id(2); i < k; i += stride) { + const float d = float(d_ptr[i / QK8_0]); + + yy[i + 0] = d * qs[i + 0]; + yy[i + 1] = d * qs[i + 1]; } - } template From 1268edac579110aeb1407f7928f53193ed4e28d4 Mon Sep 17 00:00:00 2001 From: Gabe Goodhart Date: Sat, 17 Oct 2026 01:14:32 -0600 Subject: [PATCH 193/431] model : Granite switch decision 256 options (#30693) * feat(convert): Attempt to come up with 256 single-token, semantic option markers Each switch model needs to identify a collection of single-token option markers that can be mapped to their respective logits for the output probability distribution. The simple upper/lower/numbers set only yields 66 unique markers, but the model can support up to 255. The challenge is that which combinations of characters result in single-token markers is model- specific, so we do a discovery exercise at conversion time by looking through the specific model's vocab for single-token sequences of 1+ plain letter strings (eg `A`, `AA`, `BB`, etc). Branch: GraniteSwitchDecisionFullOptions AI-usage: none Signed-off-by: Gabe Goodhart * docs: Note granite-switch with 256 option markers Branch: GraniteSwitchDecisionFullOptions AI-usage: none Signed-off-by: Gabe Goodhart * style: Remove extra newline Branch: GraniteSwitchDecisionFullOptions AI-usage: none Signed-off-by: Gabe Goodhart * feat(convert): Use the computed labels in the template so c++ side is source of truth Branch: GraniteSwitchDecisionFullOptions AI-usage: full (Claude + Sonnet 5.5) Signed-off-by: Gabe Goodhart * feat: Use LEV case for GRANITE_SWITCH w/ computed labels Branch: GraniteSwitchDecisionFullOptions AI-usage: full (Claude + Sonnet 5.5) Signed-off-by: Gabe Goodhart * docs: Consolidate g-s into 255 bucket Branch: GraniteSwitchDecisionFullOptions AI-usage: full (Claude + Sonnet 5.5) Signed-off-by: Gabe Goodhart --- conversion/granite.py | 7 ++++--- tools/server/README.md | 2 +- tools/server/server-decision.cpp | 7 +++++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/conversion/granite.py b/conversion/granite.py index d2ef7c836c8d..4c6c5a0e4cc5 100644 --- a/conversion/granite.py +++ b/conversion/granite.py @@ -296,19 +296,20 @@ def _has_decision_adapter(self) -> bool: def set_vocab(self): super().set_vocab() if self._has_decision_adapter(): - self.gguf_writer.add_chat_template([{"name": "systemone", "template": self._systemone_template()}]) + systemone_template = self._systemone_template() + logger.info("System One Template: %s", systemone_template) + self.gguf_writer.add_chat_template([{"name": "systemone", "template": systemone_template}]) def _systemone_template(self) -> str: # prompt of the decision aLoRA: a state turn, then a question turn that opens with the control token # (it replaces the first token of "### DECISION"), the answer is the label token that comes next description = jinja_str_or_json("o.description") option = ( - "{% if type != 'noul' %}{{ letters[loop.index0] }}: {% if o.description %}" + description + "{% endif %}" + "{% if type != 'noul' %}{{ o.label }}: {% if o.description %}" + description + "{% endif %}" "{% elif o.key == 'true' %}A: {% if o.description %}" + description + "{% else %}The statement is true.{% endif %}" "{% else %}B: {% if o.description %}" + description + "{% else %}The statement is false.{% endif %}{% endif %}" ) return ( - "{% set letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' %}" "<|start_of_role|>user<|end_of_role|>State:\n" + jinja_str_or_json("state") + "<|end_of_text|>\n" "<|start_of_role|>user<|end_of_role|><|decision|> DECISION\nQuestion:\n" + jinja_str_or_json("instructions") + "{% if type == 'score' %} Rate along the ordered levels below (lowest first).{% endif %}" diff --git a/tools/server/README.md b/tools/server/README.md index e4a0c418bf10..c42336c7f217 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -1723,7 +1723,7 @@ Follows the [TypeSafe API](https://docs.typesafe.ai/api), streaming is not suppo The questions of a request are answered independently, an answer does not depend on the other questions. The exception is clef: it reads all the questions in one prompt and decides them jointly. -The number of options of a `choice` question is limited by the model, for example: 52 for openjev and granite-switch, 255 for laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with. +The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for granite-switch, laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with. For laya, clef, vllm-d3 and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. A server that runs clef only serves this endpoint, text generation is not available. A decision model that is also a text model, such as granite-switch, serves both and reports `["text", "decisions"]`. diff --git a/tools/server/server-decision.cpp b/tools/server/server-decision.cpp index 3905c81b34c3..be9bc8ef4912 100644 --- a/tools/server/server-decision.cpp +++ b/tools/server/server-decision.cpp @@ -87,7 +87,7 @@ void server_decision_context::init(const llama_model * model) { bias[level] = std::strtof(val, nullptr); } - if (model_type == COMMON_DECISION_TYPE_OPENJEV || model_type == COMMON_DECISION_TYPE_GRANITE_SWITCH) { + if (model_type == COMMON_DECISION_TYPE_OPENJEV) { // one letter per option, each must be a single token const std::string letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; for (const char c : letters) { @@ -99,7 +99,7 @@ void server_decision_context::init(const llama_model * model) { } n_options_max = labels.size(); noul_true_first = true; - } else if (model_type == COMMON_DECISION_TYPE_LEV || model_type == COMMON_DECISION_TYPE_NIMBLE || model_type == COMMON_DECISION_TYPE_PPLX_DECIDER || model_type == COMMON_DECISION_TYPE_VLLM_D3) { + } else if (model_type == COMMON_DECISION_TYPE_LEV || model_type == COMMON_DECISION_TYPE_GRANITE_SWITCH || model_type == COMMON_DECISION_TYPE_NIMBLE || model_type == COMMON_DECISION_TYPE_PPLX_DECIDER || model_type == COMMON_DECISION_TYPE_VLLM_D3) { // label codes are A..Z then AA..ZZ, only the ones that are a single token are used std::vector codes; for (char a = 'A'; a <= 'Z'; a++) { @@ -121,6 +121,9 @@ void server_decision_context::init(const llama_model * model) { if (model_type == COMMON_DECISION_TYPE_VLLM_D3) { canonical_legend = true; } + if (model_type == COMMON_DECISION_TYPE_GRANITE_SWITCH) { + noul_true_first = true; + } } else if (model_type == COMMON_DECISION_TYPE_KEV) { // the hidden state of an option is read at the token that ends it const auto toks = common_tokenize(vocab, "<|box_end|>", false, true); From 83f9389419ef93117d1444f87adfcd9f03ec0509 Mon Sep 17 00:00:00 2001 From: Sandro Hanea <40202887+sandrohanea@users.noreply.github.com> Date: Sat, 17 Oct 2026 11:41:53 +0200 Subject: [PATCH 194/431] ci : added ABI reports as github artifact (#30619) --- .github/workflows/make-release.yml | 9 +++++++++ scripts/check-release-apiabi.sh | 14 +++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/.github/workflows/make-release.yml b/.github/workflows/make-release.yml index 57c3a04a577f..2e5e44caaab8 100644 --- a/.github/workflows/make-release.yml +++ b/.github/workflows/make-release.yml @@ -61,8 +61,17 @@ jobs: RELEASE_BRANCH: ${{ github.ref_name }} SKIP_APIABI_CHECK: ${{ github.event.inputs.skip_apiabi_check }} APIABI_COMPARE_TAG: ${{ github.event.inputs.apiabi_compare_tag }} + APIABI_REPORT_DIR: ${{ runner.temp }}/apiabi-reports REQUIRE_DOCKER: ${{ github.event.inputs.require_docker }} + - name: Upload API/ABI reports + if: ${{ always() && github.event.inputs.skip_apiabi_check != 'true' }} + uses: actions/upload-artifact@v7 + with: + name: apiabi-reports + path: ${{ runner.temp }}/apiabi-reports + if-no-files-found: ignore + - name: Create release tag if: ${{ github.event.inputs.dry_run == 'false' }} run: | diff --git a/scripts/check-release-apiabi.sh b/scripts/check-release-apiabi.sh index f4803adc5155..c624d12c5f10 100755 --- a/scripts/check-release-apiabi.sh +++ b/scripts/check-release-apiabi.sh @@ -15,6 +15,7 @@ # Environment: # SKIP_APIABI_CHECK: set to 1 or true to skip # APIABI_COMPARE_TAG: equivalent to --tag (used by CI) +# APIABI_REPORT_DIR: preserve reports and dumps in this directory, even if the comparison fails set -euo pipefail @@ -113,4 +114,15 @@ cmake --build "$BUILD_NEW" --parallel "$(nproc)" (cd "$WORKTREE_DIR" && "$SCRIPT_DIR/check-apiabi-compat.sh" --include-path ggml/include --generate "$BUILD_OLD" "${LIBS[@]}") (cd "$REPO_ROOT" && "$SCRIPT_DIR/check-apiabi-compat.sh" --include-path ggml/include --generate "$BUILD_NEW" "${LIBS[@]}") -(cd "$REPO_ROOT" && "$SCRIPT_DIR/check-apiabi-compat.sh" "${CHECK_FLAGS[@]}" --check "$BUILD_OLD" "$BUILD_NEW") + +CHECK_RESULT=0 +(cd "$REPO_ROOT" && "$SCRIPT_DIR/check-apiabi-compat.sh" "${CHECK_FLAGS[@]}" --check "$BUILD_OLD" "$BUILD_NEW") || CHECK_RESULT=$? + +if [[ -n "${APIABI_REPORT_DIR:-}" ]]; then + mkdir -p "$APIABI_REPORT_DIR/old" "$APIABI_REPORT_DIR/new" + cp -R "$BUILD_OLD/apiabi/." "$APIABI_REPORT_DIR/old/" + cp -R "$BUILD_NEW/apiabi/." "$APIABI_REPORT_DIR/new/" + echo "API/ABI reports saved to $APIABI_REPORT_DIR" +fi + +exit "$CHECK_RESULT" From 62318cd57dc63900ecb153272c0b10f78550855d Mon Sep 17 00:00:00 2001 From: Xue Zhenliang Date: Sat, 17 Oct 2026 19:00:17 +0800 Subject: [PATCH 195/431] model : add Kolibri-1 (Aleph-Alpha) support (#30673) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * model : add Kolibri-1 (Aleph-Alpha) support Kolibri-1 is a bilingual German/English MoE reasoning model with 78B parameters and 3.46B active per token: 50 layers, 384 routed experts plus one shared expert. Attention interleaves four sliding layers at a 513 window with one full layer. The sliding layers rotate with NeoX RoPE, the full layers are NoPE. GQA runs 48 query heads against 4 KV heads at head_dim 128 with a QK-RMSNorm, and every block carries four RMSNorms, so the attention and the MLP output are normalized before they rejoin the residual stream. Routing picks the experts from logits + expert_bias and weights them with the sigmoid of the unbiased logits, without renorm. The converter reuses AfmoeModel for the per-expert checkpoint layout and the expert selection bias, and writes the shared expert and the per layer sliding window pattern. The released FP8 checkpoint needs no special code, it goes through the generic weight_scale_inv path. Three tensor names are added to the mapping (post_attn_norm, post_ffn_norm, moe.router.expert_bias) and the 128k vocabulary registers the qwen2 pre-tokenizer by hash. The arch is registered in test-llama-archs, with GQA and the sliding window pattern. Assisted-by: DeepSeek Harness (DeepSeek V4.1 Flash) * Update src/models/kolibri1.cpp Co-authored-by: Sigbjørn Skjæret * Add fusion records for kolibri1 * Update src/models/kolibri1.cpp Co-authored-by: Sigbjørn Skjæret --------- Co-authored-by: Sigbjørn Skjæret --- conversion/__init__.py | 1 + conversion/base.py | 3 + conversion/kolibri1.py | 23 ++++ convert_hf_to_gguf_update.py | 2 + gguf-py/gguf/constants.py | 25 +++++ gguf-py/gguf/tensor_mapping.py | 3 + src/llama-arch.cpp | 1 + src/llama-arch.h | 1 + src/llama-model.cpp | 4 + src/llama-model.h | 1 + src/models/kolibri1.cpp | 187 +++++++++++++++++++++++++++++++++ src/models/models.h | 13 +++ tests/fusion/MTL.csv | 3 + tests/test-llama-archs.cpp | 10 +- 14 files changed, 275 insertions(+), 2 deletions(-) create mode 100644 conversion/kolibri1.py create mode 100644 src/models/kolibri1.cpp diff --git a/conversion/__init__.py b/conversion/__init__.py index 01c8db29680c..e3c10d5fc54e 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -146,6 +146,7 @@ "JinaBertModel": "bert", "JinaEmbeddingsV5Model": "bert", "K2HorizonForCausalLM": "k2_horizon", + "Kolibri1ForCausalLM": "kolibri1", "KORMoForCausalLM": "qwen", "KimiK25ForConditionalGeneration": "deepseek", "KimiK3ForConditionalGeneration": "kimi_k3", diff --git a/conversion/base.py b/conversion/base.py index e3d265157add..72c1256ba74f 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -1872,6 +1872,9 @@ def get_vocab_base_pre(self, tokenizer) -> str: if chkhsh == "f5c965e4198da6e7525072d87d1ecf4c51d981fd486c7c4ec728ef9cec7e03fc": # ref: https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B res = "gpt-4o" + if chkhsh == "6e040dfe72e4b85855588c53acf4909ac4e98e55bc3a33cd7db499180dc42a78": + # ref: https://huggingface.co/Aleph-Alpha/Kolibri-1 + res = "qwen2" if chkhsh == "b3d1dd861f1d4c5c0d2569ce36baf3f90fe8a102db3de50dd71ff860d91be3df": # ref: https://huggingface.co/aari1995/German_Semantic_V3 res = "jina-v2-de" diff --git a/conversion/kolibri1.py b/conversion/kolibri1.py new file mode 100644 index 000000000000..845501f4ef38 --- /dev/null +++ b/conversion/kolibri1.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from .afmoe import AfmoeModel +from .base import ModelBase, gguf + + +@ModelBase.register("Kolibri1ForCausalLM") +@ModelBase.example("Aleph-Alpha/Kolibri-1") +class Kolibri1Model(AfmoeModel): + model_arch = gguf.MODEL_ARCH.KOLIBRI1 + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # AfmoeModel reads these keys, this config stores the same values under other names + self.hparams["num_shared_experts"] = 1 + self.hparams["route_norm"] = self.hparams["norm_topk_prob"] + self.hparams["intermediate_size"] = self.hparams["moe_intermediate_size"] + + def set_gguf_parameters(self): + super().set_gguf_parameters() + self.gguf_writer.add_expert_shared_feed_forward_length(self.hparams["shared_expert_intermediate_size"]) + self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in self.hparams["layer_types"]]) + self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID) diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py index d49fb8f3ff3e..d3aa3c3807f0 100755 --- a/convert_hf_to_gguf_update.py +++ b/convert_hf_to_gguf_update.py @@ -191,6 +191,8 @@ class TOKENIZER_TYPE(IntEnum): {"name": "grok-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/alvarobartt/grok-2-tokenizer", "chkhsh": "66b8d4e19ab16c3bfd89bce5d785fb7e0155e8648708a1f42077cb9fe002c273"}, # muse-spark uses the l4_202k tokenizer, whose pre-tokenizer regex is gpt-4o's {"name": "gpt-4o", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B", "chkhsh": "f5c965e4198da6e7525072d87d1ecf4c51d981fd486c7c4ec728ef9cec7e03fc"}, + # kolibri1 uses a 128k vocabulary with the qwen2 pre-tokenizer regex + {"name": "qwen2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Aleph-Alpha/Kolibri-1", "chkhsh": "6e040dfe72e4b85855588c53acf4909ac4e98e55bc3a33cd7db499180dc42a78"}, # jina-v2-de variants {"name": "jina-v2-de", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/aari1995/German_Semantic_V3", "chkhsh": "b3d1dd861f1d4c5c0d2569ce36baf3f90fe8a102db3de50dd71ff860d91be3df"}, {"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/evilfreelancer/ruGPT3XL", "chkhsh": "0fe1cf6eda062318a1af7270f3331a85c539a01778ff948e24388e949c5282f4"}, diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index bfeab6adfde6..32e088d8820a 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -701,6 +701,7 @@ class MODEL_ARCH(IntEnum): QWEN3TTS = auto() POCKETTTS = auto() K2HORIZON = auto() + KOLIBRI1 = auto() class VISION_PROJECTOR_TYPE(IntEnum): @@ -1518,6 +1519,7 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH.QWEN3TTS: "qwen3tts", MODEL_ARCH.POCKETTTS: "pockettts", MODEL_ARCH.K2HORIZON: "k2-horizon", + MODEL_ARCH.KOLIBRI1: "kolibri1", } VISION_PROJECTOR_TYPE_NAMES: dict[VISION_PROJECTOR_TYPE, str] = { @@ -6053,6 +6055,29 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_UP_SHEXP, MODEL_TENSOR.FFN_DOWN_SHEXP, ], + MODEL_ARCH.KOLIBRI1: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_POST_NORM, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_Q_NORM, + MODEL_TENSOR.ATTN_K_NORM, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_POST_NORM, + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_EXP_PROBS_B, + MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_UP_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_GATE_SHEXP, + MODEL_TENSOR.FFN_UP_SHEXP, + MODEL_TENSOR.FFN_DOWN_SHEXP, + ], } # tensors that will not be serialized diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index ab6abf9cb704..3586d65285ca 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -416,6 +416,7 @@ class TensorNameMap: "model.layers.{bid}.post_attention_layernorm", # gemma2 olmo2 # ge "layers.{bid}.post_attention_layernorm", # embeddinggemma "model.layers.{bid}.post_self_attn_layernorm", # glm-4-0414 + "model.layers.{bid}.post_attn_norm", # kolibri1 "model.layers.layers.{bid}.post_mixer_norm.weight", # plamo2 ), @@ -498,6 +499,7 @@ class TensorNameMap: "model.layers.{bid}.post_feedforward_layernorm", # gemma2 olmo2 "layers.{bid}.post_feedforward_layernorm", # embeddinggemma "model.layers.{bid}.post_mlp_layernorm", # glm-4-0414 + "model.layers.{bid}.post_ffn_norm", # kolibri1 "model.layers.layers.{bid}.post_mlp_norm.weight", # plamo2 "model.layers.{bid}.feed_forward.up_proj", "model.layers.{bid}.post_moe_norm", # grok-2 @@ -542,6 +544,7 @@ class TensorNameMap: "model.layers.{bid}.mlp.moe_statics.e_score_correction", # ernie4.5-moe "model.layers.{bid}.mlp.gate.expert_bias", # bailingmoe2 "model.layers.{bid}.mlp.expert_bias", # afmoe + "model.layers.{bid}.moe.router.expert_bias", # kolibri1 "model.layers.{bid}.feed_forward.expert_bias", # lfm2moe "model.layers.{bid}.block_sparse_moe.e_score_correction", # minimax-m2 "backbone.layers.{bid}.mixer.gate.e_score_correction", # nemotron-h-moe diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index bef2d45c0ba2..ffa35bdea7d4 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -166,6 +166,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_POCKETTTS, "pockettts" }, { LLM_ARCH_K2_HORIZON, "k2-horizon" }, { LLM_ARCH_BEAM, "beam" }, + { LLM_ARCH_KOLIBRI1, "kolibri1" }, { LLM_ARCH_UNKNOWN, "(unknown)" }, }; diff --git a/src/llama-arch.h b/src/llama-arch.h index c305c17663cb..14ce29ab3238 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -171,6 +171,7 @@ enum llm_arch { LLM_ARCH_HRM_TEXT, LLM_ARCH_K2_HORIZON, LLM_ARCH_BEAM, + LLM_ARCH_KOLIBRI1, LLM_ARCH_UNKNOWN, }; diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 099c7a052f46..803a8329e9db 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -360,6 +360,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_spark2_5(params); case LLM_ARCH_K2_HORIZON: return new llama_model_k2_horizon(params); + case LLM_ARCH_KOLIBRI1: + return new llama_model_kolibri1(params); default: throw std::runtime_error(std::string("unsupported model architecture: '") + llm_arch_name(arch) + "'"); } @@ -1002,6 +1004,7 @@ const char * llm_type_name(llm_type type) { case LLM_TYPE_10B_128x3_66B: return "10B+128x3.66B"; case LLM_TYPE_57B_A14B: return "57B.A14B"; case LLM_TYPE_816B_A42B: return "816B.A42B"; + case LLM_TYPE_78B_A3_5B: return "78B.A3.5B"; case LLM_TYPE_17B_16E: return "17Bx16E (Scout)"; case LLM_TYPE_17B_128E: return "17Bx128E (Maverick)"; case LLM_TYPE_A13B: return "A13B"; @@ -3615,6 +3618,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_MAPLE: case LLM_ARCH_HRM_TEXT: case LLM_ARCH_BEAM: + case LLM_ARCH_KOLIBRI1: return LLAMA_ROPE_TYPE_NEOX; case LLM_ARCH_DFLASH: diff --git a/src/llama-model.h b/src/llama-model.h index 6487b0e9c434..a187c274932b 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -115,6 +115,7 @@ enum llm_type { LLM_TYPE_10B_128x3_66B, LLM_TYPE_57B_A14B, LLM_TYPE_816B_A42B, + LLM_TYPE_78B_A3_5B, // Kolibri1 LLM_TYPE_17B_16E, // llama4 Scout LLM_TYPE_17B_128E, // llama4 Maverick LLM_TYPE_A13B, diff --git a/src/models/kolibri1.cpp b/src/models/kolibri1.cpp new file mode 100644 index 000000000000..acc1dd159202 --- /dev/null +++ b/src/models/kolibri1.cpp @@ -0,0 +1,187 @@ +#include "models.h" + +void llama_model_kolibri1::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); + ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); + ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm); + ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func); + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa); + + if (hparams.n_swa == 0 || hparams.n_expert_shared != 1 || hparams.n_ff_shexp == 0) { + throw std::runtime_error("Kolibri-1: invalid sliding window or shared expert configuration"); + } + + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; + ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl); + + hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train; + hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train; + + switch (hparams.n_layer()) { + case 50: type = LLM_TYPE_78B_A3_5B; break; + default: type = LLM_TYPE_UNKNOWN; + } +} + +void llama_model_kolibri1::load_arch_tensors(llama_model_loader &) { + LLAMA_LOAD_LOCALS; + + const int64_t n_ff_exp = hparams.n_ff_exp(); + const int64_t n_ff_shexp = hparams.n_ff_shexp; + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0); + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, 0); + + create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0); + + layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0); + layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0); + + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, "weight", i), {n_embd}, 0); + + // Routing is sigmoid, the selection bias is added to the logits before top-k. + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); + layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0); + + layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0); + layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0); + + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_shexp}, 0); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_shexp}, 0); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, 0); + } +} + +llama_model_kolibri1::graph::graph(const llama_model & model, const llm_graph_params & params) + : llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v(); + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); + + ggml_tensor * cur; + ggml_tensor * inpL = build_inp_embd(model.tok_embd); + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_iswa(); + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + const float kq_scale = 1.0f / sqrtf(float(n_embd_head)); + + for (int il = 0; il < n_layer; ++il) { + const auto & layer = model.layers[il]; + + ggml_tensor * residual = inpL; + + cur = build_norm(inpL, layer.attn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // QK-norm, then RoPE on the sliding layers only: the global layers are NoPE. + auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur, n_embd_head, n_head, n_head_kv, il); + + Qcur = build_norm(Qcur, layer.attn_q_norm, NULL, LLM_NORM_RMS, il); + Kcur = build_norm(Kcur, layer.attn_k_norm, NULL, LLM_NORM_RMS, il); + cb(Qcur, "Qcur_normed", il); + cb(Kcur, "Kcur_normed", il); + + if (hparams.is_swa(il)) { + const float freq_base_l = model.get_rope_freq_base (cparams, il); + const float freq_scale_l = model.get_rope_freq_scale(cparams, il); + + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(Qcur, "Qcur_rope", il); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(Kcur, "Kcur_rope", il); + } + + cur = build_attn(inp_attn, + layer.wo, NULL, layer.wo_s, + Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); + cb(cur, "attn_out", il); + + cur = build_norm(cur, layer.attn_post_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "attn_post_norm", il); + + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + residual = ggml_get_rows(ctx0, residual, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, residual); + cb(ffn_inp, "ffn_inp", il); + + cur = build_norm(ffn_inp, layer.ffn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + // The router runs in f32: the selection adds the bias to the logits, the weights + // are the sigmoid of the unbiased logits, so the same logits feed both. + ggml_tensor * logits = build_lora_mm(layer.ffn_gate_inp, cur); + ggml_prec_set_acc(logits, GGML_PREC_F32); + ggml_prec_set_src(logits, GGML_PREC_F32, 1); + cb(logits, "ffn_moe_logits", il); + + ggml_tensor * selection = ggml_add(ctx0, logits, layer.ffn_exp_probs_b); + cb(selection, "ffn_moe_logits_biased", il); + + ggml_tensor * selected = ggml_argsort_top_k(ctx0, selection, n_expert_used); + cb(selected, "ffn_moe_topk", il); + + ggml_tensor * moe_out = build_moe_ffn(cur, + layer.ffn_gate_inp, + layer.ffn_up_exps, + layer.ffn_gate_exps, + layer.ffn_down_exps, + nullptr, + n_expert, n_expert_used, + LLM_FFN_SILU, hparams.expert_weights_norm, 1.0f, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il, + logits, + nullptr, nullptr, nullptr, nullptr, selected); + cb(moe_out, "ffn_moe_out", il); + + ggml_tensor * shared = build_ffn(cur, + layer.ffn_up_shexp, nullptr, nullptr, + layer.ffn_gate_shexp, nullptr, nullptr, + layer.ffn_down_shexp, nullptr, nullptr, nullptr, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(shared, "ffn_shexp_out", il); + + cur = ggml_add(ctx0, moe_out, shared); + cur = build_norm(cur, layer.ffn_post_norm, NULL, LLM_NORM_RMS, il); + cur = ggml_add(ctx0, cur, ffn_inp); + + inpL = build_cvec(cur, il); + cb(inpL, "l_out", il); + } + + cur = build_norm(inpL, model.output_norm, NULL, LLM_NORM_RMS, -1); + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur, model.output_s); + cb(cur, "result_output", -1); + res->t_logits = cur; +} + +std::unique_ptr llama_model_kolibri1::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique(*this, params); +} diff --git a/src/models/models.h b/src/models/models.h index 98f6b6e35dcc..e6cf9d213cee 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -2974,3 +2974,16 @@ struct llama_model_k2_horizon : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; + + +struct llama_model_kolibri1 : public llama_model_base { + llama_model_kolibri1(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + struct graph : public llm_graph_context { + graph(const llama_model & model, const llm_graph_params & params); + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv index d31a9f23e5f0..9ab1f1f3f885 100644 --- a/tests/fusion/MTL.csv +++ b/tests/fusion/MTL.csv @@ -139,6 +139,9 @@ kimi-linear ,0 ,any ,GATED_DELTA_NET+CPY , 1 kimi-linear ,0 ,any ,MUL+ADD , 2 kimi-linear ,0 ,any ,RMS_NORM+MUL , 7 kimi-linear ,0 ,any ,RMS_NORM+SCALE , 2 +kolibri1 ,0 ,any ,MUL+ADD , 4 +kolibri1 ,0 ,any ,RMS_NORM+MUL , 10 +kolibri1 ,0 ,any ,RMS_NORM+MUL+ADD , 3 laguna ,0 ,any ,ADD+ADD , 1 laguna ,0 ,any ,MUL+ADD , 2 laguna ,0 ,any ,RMS_NORM+MUL , 9 diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 14ca203be809..143c1335e4ba 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -185,7 +185,8 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { uint32_t n_head_kv = n_head; if (arch == LLM_ARCH_QWEN3) { n_head_kv = 1; // MQA coverage - } else if (arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_AFMOE || arch == LLM_ARCH_MAION_CODER) { + } else if (arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_AFMOE || arch == LLM_ARCH_MAION_CODER || + arch == LLM_ARCH_KOLIBRI1) { n_head_kv = 2; // GQA coverage } if (arch == LLM_ARCH_BEAM) { @@ -316,7 +317,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(5)); } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_SPARK2_5 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_GRANITE_SWA || arch == LLM_ARCH_DOTS3NOTE || - arch == LLM_ARCH_MAPLE) { + arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_MAPLE) { std::vector pattern; pattern.reserve(n_layer); for (uint32_t il = 0; il < n_layer; il++) { @@ -336,6 +337,10 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true); } + if (arch == LLM_ARCH_KOLIBRI1) { + ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, false); + } + // MSA requires one indexer head per GQA (KV) head, unlike the DSA archs where the // indexer head count is independent of the main attention head count. if (arch == LLM_ARCH_QWEN4EXP || arch == LLM_ARCH_GLM5_NEXT) { @@ -758,6 +763,7 @@ static bool moe_mandatory(const llm_arch arch) { case LLM_ARCH_LAGUNA: case LLM_ARCH_MAPLE: case LLM_ARCH_MUSE_SPARK: + case LLM_ARCH_KOLIBRI1: return true; default: return false; From f2f667b02dc76d675395ae3b14546c8108d26387 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Sat, 17 Oct 2026 13:55:33 +0200 Subject: [PATCH 196/431] cuda : add grouped experts top-k fusion (#29181) * fuse grouped experts * add tests * hip fix * add to graph_optimize * apply suggestion from am17an * hip fix * fix hip fix * fix fix fix * gate on hip version instead --- ggml/src/ggml-cuda/ggml-cuda.cu | 101 +++++++++++++++++++++++++++----- ggml/src/ggml-cuda/topk-moe.cu | 66 +++++++++++++++++++++ ggml/src/ggml-cuda/topk-moe.cuh | 2 + tests/test-backend-ops.cpp | 38 +++++++++++- 4 files changed, 188 insertions(+), 19 deletions(-) diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 6fa22289f331..cb85321f5762 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -3029,6 +3029,65 @@ static int ggml_cuda_try_ssm_scan_cache_fusion( return skip; } +// expert group selection from build_moe_ffn (n_expert_groups > 1) +// src0/src1 index into this table, -1 is the (biased) probs, -2 is not checked +static const struct { + ggml_op op; + int src0; + int src1; +} topk_moe_group_pattern[] = { + { GGML_OP_RESHAPE, -1, -2 }, // 0: [n_exp_per_group, n_expert_groups, n_tokens] + { GGML_OP_RESHAPE, 0, -2 }, // 1: [1, n_exp_per_group, n_expert_groups, n_tokens] + { GGML_OP_ARGSORT, 0, -2 }, // 2 + { GGML_OP_VIEW, 2, -2 }, // 3: top-2 per group + { GGML_OP_GET_ROWS, 1, 3 }, // 4 + { GGML_OP_RESHAPE, 4, -2 }, // 5 + { GGML_OP_SUM_ROWS, 5, -2 }, // 6: group scores + { GGML_OP_RESHAPE, 6, -2 }, // 7 + { GGML_OP_ARGSORT, 7, -2 }, // 8 + { GGML_OP_VIEW, 8, -2 }, // 9: top n_group_used groups + { GGML_OP_GET_ROWS, 0, 9 }, // 10 + { GGML_OP_FILL, 0, -2 }, // 11 + { GGML_OP_SET_ROWS, 10, 9 }, // 12: unused groups -> -inf + { GGML_OP_RESHAPE, 12, -2 }, // 13: [n_expert, n_tokens] +}; + +static bool ggml_cuda_topk_moe_group_fusion(const struct ggml_cgraph * cgraph, int node_idx, const ggml_tensor * selection, ggml_cuda_topk_moe_args & args) { +#if defined(GGML_USE_HIP) && HIP_VERSION < 60300000 + GGML_UNUSED_VARS(cgraph, node_idx, selection, args); + return false; +#else + const int n_ops = (int) std::size(topk_moe_group_pattern); + if (node_idx + n_ops > cgraph->n_nodes) { + return false; + } + + ggml_tensor ** nodes = cgraph->nodes + node_idx; + auto src_ok = [&](const ggml_tensor * src, int idx) { + return idx == -2 || src == (idx == -1 ? selection : nodes[idx]); + }; + + for (int i = 0; i < n_ops; i++) { + const auto & p = topk_moe_group_pattern[i]; + if (nodes[i]->op != p.op || !src_ok(nodes[i]->src[0], p.src0) || !src_ok(nodes[i]->src[1], p.src1)) { + return false; + } + } + + const int n_groups = nodes[0]->ne[1]; + const int n_group_used = nodes[9]->ne[0]; + + if (nodes[3]->ne[0] != 2 || nodes[12]->src[2] != nodes[11] || ggml_get_op_params_f32(nodes[11], 0) != -INFINITY || + n_groups > WARP_SIZE) { + return false; + } + + args.n_groups = n_groups; + args.n_group_used = n_group_used; + return true; +#endif // defined(GGML_USE_HIP) && HIP_VERSION < 60300000 +} + static bool ggml_cuda_topk_moe_fusion(const struct ggml_cgraph * cgraph, int node_idx, ggml_cuda_topk_moe_args & args) { args.sigmoid = false; args.sqrt_softplus = false; @@ -3036,6 +3095,8 @@ static bool ggml_cuda_topk_moe_fusion(const struct ggml_cgraph * cgraph, int nod args.delayed_softmax = false; args.prob_bias = false; args.norm = false; + args.n_groups = 0; + args.n_group_used = 0; const int n_nodes = cgraph->n_nodes; ggml_tensor ** nodes = cgraph->nodes; @@ -3078,18 +3139,20 @@ static bool ggml_cuda_topk_moe_fusion(const struct ggml_cgraph * cgraph, int nod } // src of bias add is the unreshaped probs (-2 instead of -1) - if (nodes[node_idx]->op == GGML_OP_ADD && nodes[node_idx]->src[0] == nodes[node_idx - 2]) { + const ggml_tensor * selection = nodes[node_idx - 2]; + if (nodes[node_idx]->op == GGML_OP_ADD && nodes[node_idx]->src[0] == selection) { args.prob_bias = true; + selection = nodes[node_idx]; node_idx++; } - // RESHAPE/ADD -> ARGSORT - if (node_idx >= n_nodes || nodes[node_idx]->op != GGML_OP_ARGSORT) { - return false; + + if (ggml_cuda_topk_moe_group_fusion(cgraph, node_idx, selection, args)) { + node_idx += (int) std::size(topk_moe_group_pattern); + selection = nodes[node_idx - 1]; } - if (args.prob_bias && nodes[node_idx]->src[0] != nodes[node_idx - 1]) { - return false; - } else if (!args.prob_bias && nodes[node_idx]->src[0] != nodes[node_idx - 2]) { + // RESHAPE/ADD/group mask -> ARGSORT + if (node_idx >= n_nodes || nodes[node_idx]->op != GGML_OP_ARGSORT || nodes[node_idx]->src[0] != selection) { return false; } @@ -3100,6 +3163,11 @@ static bool ggml_cuda_topk_moe_fusion(const struct ggml_cgraph * cgraph, int nod nodes[node_idx]->src[0] != nodes[node_idx - 1]) { return false; } + + // kept groups must hold all used experts + if (args.n_groups > 0 && nodes[node_idx]->ne[0] > args.n_group_used * (probs_reshaped->ne[1] / args.n_groups)) { + return false; + } node_idx++; if (node_idx >= n_nodes || nodes[node_idx]->op != GGML_OP_GET_ROWS) { @@ -3754,17 +3822,18 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph } else { ops.insert(ops.end(), { GGML_OP_SOFT_MAX }); } - const int i_probs = i + (int) ops.size() - 1; // last node of the gating activation - + ops.push_back(GGML_OP_RESHAPE); if (args.prob_bias) { - bias = cgraph->nodes[i_probs + 2]->src[1]; - ops.insert(ops.end(), { GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_ARGSORT, GGML_OP_VIEW, - GGML_OP_GET_ROWS }); - out_nodes[0] = i_probs + 4; - } else { - ops.insert(ops.end(), { GGML_OP_RESHAPE, GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS }); - out_nodes[0] = i_probs + 3; + ops.push_back(GGML_OP_ADD); + bias = cgraph->nodes[i + ops.size() - 1]->src[1]; + } + if (args.n_groups > 0) { + for (const auto & p : topk_moe_group_pattern) { + ops.push_back(p.op); + } } + ops.insert(ops.end(), { GGML_OP_ARGSORT, GGML_OP_VIEW, GGML_OP_GET_ROWS }); + out_nodes[0] = i + ops.size() - 2; ids = cgraph->nodes[out_nodes[0]]; if (args.norm) { diff --git a/ggml/src/ggml-cuda/topk-moe.cu b/ggml/src/ggml-cuda/topk-moe.cu index d015e7ddefa5..81d6ec3f7cdc 100644 --- a/ggml/src/ggml-cuda/topk-moe.cu +++ b/ggml/src/ggml-cuda/topk-moe.cu @@ -11,6 +11,8 @@ struct topk_moe_config { bool use_sqrt_softplus; bool with_norm; bool delayed_softmax; + int n_groups; + int n_group_used; }; // Warp-local softmax used for both the pre-top-k logits and the post-top-k delayed path. @@ -78,6 +80,53 @@ __device__ void sqrt_softplus_warp_inplace(float (&vals)[experts_per_thread], co } } +#if !defined(GGML_USE_HIP) || HIP_VERSION >= 60300000 +// Set -inf for experts outside the top n_group_used groups. Group score is the sum of its top-2 experts (DeepSeek V3). +template +__device__ void group_mask_warp_inplace(float (&vals)[experts_per_thread], const int n_groups, const int n_group_used, const int lane) { + const int n_per_group = n_experts / n_groups; + + float score = -INFINITY; // lane g holds the score of group g + for (int g = 0; g < n_groups; g++) { + float top1 = -INFINITY; + float top2 = -INFINITY; +#pragma unroll + for (int i = 0; i < experts_per_thread; i++) { + if ((i * WARP_SIZE + lane) / n_per_group == g) { + top2 = fmaxf(top2, fminf(top1, vals[i])); + top1 = fmaxf(top1, vals[i]); + } + } +#pragma unroll + for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1) { + const float o1 = __shfl_xor_sync(0xFFFFFFFF, top1, offset, WARP_SIZE); + const float o2 = __shfl_xor_sync(0xFFFFFFFF, top2, offset, WARP_SIZE); + top2 = fmaxf(fminf(top1, o1), fmaxf(top2, o2)); + top1 = fmaxf(top1, o1); + } + if (lane == g) { + score = top1 + top2; + } + } + + // rank of each group, lower index wins ties + int rank = 0; + for (int g = 0; g < n_groups; g++) { + const float other = __shfl_sync(0xFFFFFFFF, score, g, WARP_SIZE); + rank += other > score || (other == score && g < lane); + } + const auto keep = __ballot_sync(0xFFFFFFFFULL, lane < n_groups && rank < n_group_used); + +#pragma unroll + for (int i = 0; i < experts_per_thread; i++) { + const int g = (i * WARP_SIZE + lane) / n_per_group; + if (g >= n_groups || !((keep >> g) & 1)) { + vals[i] = -INFINITY; + } + } +} +#endif // !defined(GGML_USE_HIP) || HIP_VERSION >= 60300000 + /* This kernel does the following: 1. optionally softmax over the logits per token [n_experts, n_tokens] @@ -171,6 +220,17 @@ __global__ void topk_moe_cuda(const float * logits, } } +#if !defined(GGML_USE_HIP) || HIP_VERSION >= 60300000 + // without bias, masking wt is safe: the matcher ensures the kept groups hold all used experts + if (config.n_groups > 0) { + if constexpr (has_bias) { + group_mask_warp_inplace(selection_wt, config.n_groups, config.n_group_used, threadIdx.x); + } else { + group_mask_warp_inplace(wt, config.n_groups, config.n_group_used, threadIdx.x); + } + } +#endif // !defined(GGML_USE_HIP) || HIP_VERSION >= 60300000 + //at this point, each thread holds either a portion of the softmax distribution //or the raw logits. We do the argmax reduce over n_expert_used, each time marking //the expert weight as -inf to exclude from the next iteration @@ -297,6 +357,10 @@ static void launch_topk_moe_cuda(ggml_backend_cuda_context & ctx, cudaStream_t stream = ctx.stream(); const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, stream); +#if defined(GGML_USE_HIP) && HIP_VERSION < 60300000 + GGML_ASSERT(config.n_groups == 0 && "expert groups fusion is not supported on this HIP version"); +#endif + switch (n_expert) { case 1: ggml_cuda_kernel_launch(topk_moe_cuda<1, has_bias>, launch_params, @@ -389,6 +453,8 @@ void ggml_cuda_op_topk_moe(ggml_backend_cuda_context & ctx, config.use_sqrt_softplus = args.sqrt_softplus; config.with_norm = with_norm; config.delayed_softmax = args.delayed_softmax; + config.n_groups = args.n_groups; + config.n_group_used = args.n_group_used; if (bias) { launch_topk_moe_cuda(ctx, logits_d, weights_d, ids_d, bias_d, n_rows, n_experts, n_expert_used, clamp_val, diff --git a/ggml/src/ggml-cuda/topk-moe.cuh b/ggml/src/ggml-cuda/topk-moe.cuh index 061b37e2971e..3637500ff99d 100644 --- a/ggml/src/ggml-cuda/topk-moe.cuh +++ b/ggml/src/ggml-cuda/topk-moe.cuh @@ -14,6 +14,8 @@ struct ggml_cuda_topk_moe_args { bool prob_bias{}; bool norm{}; bool scale{}; + int n_groups{}; // > 0: pick experts only from the top n_group_used expert groups + int n_group_used{}; }; void ggml_cuda_op_topk_moe(ggml_backend_cuda_context & ctx, diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index b50364660ed9..8826b274b968 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -7328,6 +7328,7 @@ struct test_topk_moe : public test_case { const bool bias_probs; const MoeGatingFunc gating_func; const float scale_w; + const int n_expert_groups; ggml_tensor * weights {}; ggml_tensor * selected_experts {}; @@ -7336,17 +7337,21 @@ struct test_topk_moe : public test_case { bool with_norm = false, bool bias_probs = false, MoeGatingFunc gating_func = GATING_FUNC_SOFTMAX, - float scale_w = 0.0f) : + float scale_w = 0.0f, + int n_expert_groups = 1) : ne(ne), n_expert_used(n_expert_used), with_norm(with_norm), bias_probs(bias_probs), gating_func(gating_func), - scale_w(scale_w) { + scale_w(scale_w), + n_expert_groups(n_expert_groups) { GGML_ASSERT(n_expert_used <= ne[0]); + GGML_ASSERT(ne[0] % n_expert_groups == 0); + GGML_ASSERT(n_expert_used <= ne[0] / n_expert_groups); } - std::string vars() override { return VARS_TO_STR6(ne, n_expert_used, with_norm, bias_probs, gating_func, scale_w); } + std::string vars() override { return VARS_TO_STR7(ne, n_expert_used, with_norm, bias_probs, gating_func, scale_w, n_expert_groups); } std::string op_desc(ggml_tensor * t) override { GGML_UNUSED(t); @@ -7358,6 +7363,8 @@ struct test_topk_moe : public test_case { ggml_tensor * build_graph(ggml_context * ctx) override { const int n_expert = ne[0]; const int n_tokens = ne[1]; + const int n_exp_per_group = n_expert / n_expert_groups; + const int n_group_used = n_expert_groups / 2; ggml_tensor * logits = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne.data()); ggml_tensor * probs = @@ -7374,6 +7381,24 @@ struct test_topk_moe : public test_case { ggml_set_name(selection_probs, "selection_probs"); } + if (n_expert_groups > 1) { + ggml_tensor * selection_groups = ggml_reshape_3d(ctx, selection_probs, n_exp_per_group, n_expert_groups, n_tokens); // [n_exp_per_group, n_expert_groups, n_tokens] + + ggml_tensor * group_scores = ggml_argsort_top_k(ctx, selection_groups, 2); // [2, n_expert_groups, n_tokens] + group_scores = ggml_get_rows(ctx, ggml_reshape_4d(ctx, selection_groups, 1, selection_groups->ne[0], selection_groups->ne[1], selection_groups->ne[2]), group_scores); // [1, 2, n_expert_groups, n_tokens] + + group_scores = ggml_sum_rows(ctx, ggml_reshape_3d(ctx, group_scores, group_scores->ne[1], group_scores->ne[2], group_scores->ne[3])); // [1, n_expert_groups, n_tokens] + group_scores = ggml_reshape_2d(ctx, group_scores, group_scores->ne[1], group_scores->ne[2]); // [n_expert_groups, n_tokens] + + ggml_tensor * expert_groups = ggml_argsort_top_k(ctx, group_scores, n_group_used); // [n_group_used, n_tokens] + ggml_set_name(expert_groups, "ffn_moe_group_topk"); + + selection_probs = ggml_get_rows(ctx, selection_groups, expert_groups); // [n_exp_per_group, n_group_used, n_tokens] + selection_probs = ggml_set_rows(ctx, ggml_fill(ctx, selection_groups, -INFINITY), selection_probs, expert_groups); // [n_exp_per_group, n_expert_groups, n_tokens] + selection_probs = ggml_reshape_2d(ctx, selection_probs, n_expert, n_tokens); // [n_expert, n_tokens] + ggml_set_name(selection_probs, "ffn_moe_probs_masked"); + } + selected_experts = ggml_argsort_top_k(ctx, selection_probs, n_expert_used); // [n_expert_used, n_tokens] ggml_set_name(selected_experts, "selected_experts"); @@ -11937,6 +11962,13 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_topk_moe({32, 8, 1, 1}, 4, with_norm, bias_probs, gate, scale_w)); test_cases.emplace_back(new test_topk_moe({32, 8, 1, 1}, 8, with_norm, bias_probs, gate, scale_w)); test_cases.emplace_back(new test_topk_moe({32, 9, 1, 1}, 8, with_norm, bias_probs, gate, scale_w)); + + // grouped experts + test_cases.emplace_back(new test_topk_moe({128, 1, 1, 1}, 8, with_norm, bias_probs, gate, scale_w, 16)); + test_cases.emplace_back(new test_topk_moe({256, 22, 1, 1}, 6, with_norm, bias_probs, gate, scale_w, 32)); + test_cases.emplace_back(new test_topk_moe({256, 7, 1, 1}, 8, with_norm, bias_probs, gate, scale_w, 8)); // DeepSeek V3 + test_cases.emplace_back(new test_topk_moe({128, 5, 1, 1}, 6, with_norm, bias_probs, gate, scale_w, 8)); + test_cases.emplace_back(new test_topk_moe({512, 3, 1, 1}, 8, with_norm, bias_probs, gate, scale_w, 4)); } } } From 9d1e03616efb7520514d65552c7376296ef9208f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Gallou=C3=ABt?= Date: Sat, 17 Oct 2026 14:36:17 +0200 Subject: [PATCH 197/431] cuda : fix -Wtautological-compare in fattn vec dispatch (#30519) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Adrien Gallouët --- ggml/src/ggml-cuda/fattn.cu | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu index 5fa3e530840c..1515572fe111 100644 --- a/ggml/src/ggml-cuda/fattn.cu +++ b/ggml/src/ggml-cuda/fattn.cu @@ -417,9 +417,7 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg #define FATTN_VEC_CASE(D, type_K_case, type_V_case) \ if constexpr (GGML_CUDA_FA_##type_K_case##_##type_V_case) { \ - const bool type_K_okay = type_K == GGML_TYPE_##type_K_case || (type_K == GGML_TYPE_F32 && GGML_TYPE_##type_K_case == GGML_TYPE_F16); \ - const bool type_V_okay = type_V == GGML_TYPE_##type_V_case || (type_V == GGML_TYPE_F32 && GGML_TYPE_##type_V_case == GGML_TYPE_F16); \ - if (head_size == (D) && type_K_okay && type_V_okay) { \ + if (head_size == (D) && type_K == GGML_TYPE_##type_K_case && type_V == GGML_TYPE_##type_V_case) { \ return ggml_cuda_flash_attn_ext_vec_case; \ } \ } \ @@ -432,7 +430,9 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg typedef void (* fattn_vec_case_t)(ggml_backend_cuda_context & ctx, ggml_tensor * dst); // Vector kernel for the given head size and K/V types, nullptr if its template instance was not compiled: -static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, const ggml_type type_K, const ggml_type type_V) { +static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, const ggml_type type_K_in, const ggml_type type_V_in) { + const ggml_type type_K = type_K_in == GGML_TYPE_F32 ? GGML_TYPE_F16 : type_K_in; + const ggml_type type_V = type_V_in == GGML_TYPE_F32 ? GGML_TYPE_F16 : type_V_in; FATTN_VEC_CASES_ALL_D(F16, F16) FATTN_VEC_CASES_ALL_D(Q4_0, F16) FATTN_VEC_CASES_ALL_D(Q4_1, F16) From 10b8b7553494f3638da7668cf17cac089229122a Mon Sep 17 00:00:00 2001 From: Pascal Date: Sat, 17 Oct 2026 15:07:55 +0200 Subject: [PATCH 198/431] tests: give the random model generator a per-tensor filler table (#30674) * tests: give the random model generator a per-tensor filler table Muse Spark makes test-llama-context fail on state_load, cp_h and cp_d on about six percent of the random seeds, with an NMSE of 1.0e-05 to 1.4e-05 against a threshold of 1.0e-05. The sliding window is what exposes it: a restored state keeps the window alone, so the attention of an SWA layer reduces over a different cell layout and the continuation is not bit exact. Every SWA model carries that, at 1e-08. Muse Spark carried it at 1e-05. The generator is what turns 1e-08 into 1e-05. A norm preserving residual gate holds alpha^2 + beta^2 == 1 and the conversion resolves both weights from one raw weight, while the generator draws every tensor on its own. The pair then scaled the stream and the branch independently, the stream lost an arbitrary factor at each layer, and the model became an amplifier. The weights the test ran on were not a Muse Spark. Per-tensor rules lived inside the write path, where a flag picked the values of a tensor inside each of the two branches that store them, so a new rule meant a new flag threaded through both. The file asks to be rid of that in a TODO of its own. Give each rule a filler, pick one from a table keyed on a substring of the tensor name, and keep a single write path behind it. Mamba moves into the table and Muse Spark joins it: the gate filler draws the raw weight once per block and resolves alpha and beta the way the conversion does. Over a thousand seeds on a CPU build the worst case drops from 1.36e-05 to 2.48e-06 and the median from 4.10e-06 to 5.48e-07, with no seed above the threshold. The other 131 models come out identical to the bit, since no other architecture carries these tensors. ctest is green on a CPU build and on a CUDA build. * tests: draw every tensor filler from fill_normal The fillers repeated the generator and the distribution each time. The normal draw now takes its key and the other fillers transform its values, so the generator lives in one place. The generated models are identical to the bit. --- tests/test-llama-archs.cpp | 71 ++++++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 19 deletions(-) diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 143c1335e4ba..94a22ff2f57f 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -53,33 +53,66 @@ struct tensor_data_params { float stdev; }; +// fills the values of one tensor, the name carries whatever the property needs +typedef void (*tensor_filler)(std::vector & dst, const char * name, const tensor_data_params & params); + +// a normal draw keyed on a name, so that tensors which share a draw ask for the same key +static void fill_normal(std::vector & dst, const char * key, const tensor_data_params & params) { + std::mt19937 gen(params.seed ^ std::hash{}(key)); + std::normal_distribution dis(0.0f, params.stdev); + for (float & v : dst) { + v = dis(gen); + } +} + +// Mamba A must be negative (state decay) +static void fill_ssm_a(std::vector & dst, const char * name, const tensor_data_params & params) { + fill_normal(dst, name, params); + for (float & v : dst) { + v = -fabsf(v); + } +} + +// a norm preserving residual gate holds alpha^2 + beta^2 == 1, and the conversion resolves both from one raw weight +// the alpha and the beta of a block share that weight, so the key drops the suffix and keeps the block +static void fill_resid_gate(std::vector & dst, const char * name, const tensor_data_params & params) { + const char * suffix = strstr(name, "_resid_"); + const bool is_beta = strstr(suffix, "_resid_beta") != nullptr; + fill_normal(dst, std::string(name, suffix - name).c_str(), params); + for (float & v : dst) { + const float beta = 1.0f / (1.0f + expf(-v)); + v = is_beta ? beta : sqrtf(1.0f - beta*beta); + } +} + +// a tensor whose values hold a property of their own, matched on a substring of its name, first match wins +static const struct { const char * match; tensor_filler fill; } TENSOR_FILLERS[] = { + { "ssm_a", fill_ssm_a }, + { "_resid_", fill_resid_gate }, +}; + static void set_tensor_data(struct ggml_tensor * tensor, void * userdata) { const tensor_data_params & params = *(const tensor_data_params *) userdata; - size_t seed = params.seed; - std::hash hasher; - seed ^= hasher(tensor->name); - std::mt19937 gen(seed); - std::normal_distribution dis(0.0f, params.stdev); - // TODO: refactor per-tensor initialization logic in a cleaner way + tensor_filler fill = fill_normal; + for (const auto & filler : TENSOR_FILLERS) { + if (strstr(tensor->name, filler.match) != nullptr) { + fill = filler.fill; + break; + } + } + + std::vector tmp(ggml_nelements(tensor)); + fill(tmp, tensor->name, params); - // note: Mamba A must be negative (state decay) - const bool is_ssm_a = strstr(tensor->name, "ssm_a") != nullptr; - const int64_t ne = ggml_nelements(tensor); if (tensor->type == GGML_TYPE_F32) { - std::vector tmp(ne); - for (int64_t i = 0; i < ne; i++) { - float val = dis(gen); - tmp[i] = is_ssm_a ? -fabsf(val) : val; - } ggml_backend_tensor_set(tensor, tmp.data(), 0, ggml_nbytes(tensor)); } else if (tensor->type == GGML_TYPE_F16) { - std::vector tmp(ne); - for (int64_t i = 0; i < ne; i++) { - float val = dis(gen); - tmp[i] = ggml_fp32_to_fp16(is_ssm_a ? -fabsf(val) : val); + std::vector tmp16(tmp.size()); + for (size_t i = 0; i < tmp.size(); i++) { + tmp16[i] = ggml_fp32_to_fp16(tmp[i]); } - ggml_backend_tensor_set(tensor, tmp.data(), 0, ggml_nbytes(tensor)); + ggml_backend_tensor_set(tensor, tmp16.data(), 0, ggml_nbytes(tensor)); } else { GGML_ABORT("fatal error"); } From 2744e48d1e88d7416224e92bf58d4ffedda4a205 Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Sat, 17 Oct 2026 19:05:30 +0530 Subject: [PATCH 199/431] CUDA: faster MMVQ for low-bit types (#30517) * CUDA: faster MMVQ for low-bit types * ggml-cuda: hip: small nwarps GCN fix on Q3_K(ncols=2,3), Q6_K(ncols=4) --------- Co-authored-by: thelittlefireman --- ggml/src/ggml-cuda/mmvq.cu | 35 ++++++-- ggml/src/ggml-cuda/vecdotq.cuh | 149 +++++++++++++++++++++------------ 2 files changed, 125 insertions(+), 59 deletions(-) diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index 8068ba364cce..930f1adc4eee 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -468,10 +468,12 @@ static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_d } else if (table_id == MMVQ_PARAMETERS_GCN) { switch (ncols_dst) { case 1: + return 2; case 2: case 3: + return type == GGML_TYPE_Q3_K ? 1 : 2; case 4: - return 2; + return type == GGML_TYPE_Q6_K? 1 : 2; case 5: case 6: case 7: @@ -576,11 +578,31 @@ static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_d return 1; } -static constexpr __host__ __device__ int calc_rows_per_block(int ncols_dst, int table_id, bool small_k = false, int nwarps = 1) { +// Low bit types load few weight bytes per thread, a second row per block shares the q8_1 loads and adds memory parallelism. +static constexpr __host__ __device__ bool mmvq_two_rows_ncols1(ggml_type type) { + switch (type) { + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_M: + return true; + default: + return false; + } +} + +static constexpr __host__ __device__ int calc_rows_per_block(ggml_type type, int ncols_dst, int table_id, bool small_k = false, int nwarps = 1) { if (table_id == MMVQ_PARAMETERS_GENERIC || table_id == MMVQ_PARAMETERS_GCN || table_id == MMVQ_PARAMETERS_TURING || table_id == MMVQ_PARAMETERS_GB10) { switch (ncols_dst) { case 1: - return small_k ? nwarps : 1; + if (small_k) { + return nwarps; + } + if (table_id == MMVQ_PARAMETERS_GB10) { + return type == GGML_TYPE_IQ2_S || type == GGML_TYPE_IQ3_S ? 2 : 1; + } + return table_id == MMVQ_PARAMETERS_GENERIC && mmvq_two_rows_ncols1(type) ? 2 : 1; case 2: case 3: case 4: @@ -615,7 +637,7 @@ static __global__ void mul_mat_vec_q( constexpr int vdr = get_vdr_mmvq(type); constexpr mmvq_parameter_table_id table_id = get_device_table_id(); constexpr int nwarps = calc_nwarps(type, ncols_dst, table_id, small_k, halve_iters); - constexpr int rows_per_cuda_block = calc_rows_per_block(ncols_dst, table_id, small_k, nwarps); + constexpr int rows_per_cuda_block = calc_rows_per_block(type, ncols_dst, table_id, small_k, nwarps); constexpr int warp_size = ggml_cuda_get_physical_warp_size(); constexpr vec_dot_q_cuda_t vec_dot_q_cuda = get_vec_dot_q_cuda(type); @@ -727,7 +749,8 @@ static __global__ void mul_mat_vec_q( #if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK // start the next iterations' weight loads early if constexpr (mmvq_should_prefetch(type)) { - constexpr int pf_dist = 2; // loop iterations, not blocks + // loop iterations, not blocks; q3_K and q6_K keep the distance in blocks they had with vdr 1 + constexpr int pf_dist = type == GGML_TYPE_Q3_K || type == GGML_TYPE_Q6_K ? 1 : 2; const int kbx_pf = kbx + pf_dist*blocks_per_iter; if (kbx_pf < blocks_per_row_x) { #pragma unroll @@ -1013,7 +1036,7 @@ static std::pair calc_launch_params( const int ncols_dst, const int nrows_x, const int nchannels_dst, const int nsamples_or_ntokens, const int warp_size, const mmvq_parameter_table_id table_id, const bool small_k = false, const bool halve_iters = false) { const int nwarps = calc_nwarps(type, ncols_dst, table_id, small_k, halve_iters); - const int rpb = calc_rows_per_block(ncols_dst, table_id, small_k, nwarps); + const int rpb = calc_rows_per_block(type, ncols_dst, table_id, small_k, nwarps); const int64_t nblocks = (nrows_x + rpb - 1) / rpb; const dim3 block_nums(nblocks, nchannels_dst, nsamples_or_ntokens); const dim3 block_dims(warp_size, nwarps, 1); diff --git a/ggml/src/ggml-cuda/vecdotq.cuh b/ggml/src/ggml-cuda/vecdotq.cuh index f65fe530eb4f..14c1737c0207 100644 --- a/ggml/src/ggml-cuda/vecdotq.cuh +++ b/ggml/src/ggml-cuda/vecdotq.cuh @@ -360,12 +360,12 @@ static __device__ __forceinline__ float vec_dot_nvfp4_q8_1( return sum; } -#define VDR_Q2_K_Q8_1_MMVQ 1 +#define VDR_Q2_K_Q8_1_MMVQ 2 #define VDR_Q2_K_Q8_1_MMQ 4 // contiguous v/x values static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq( - const int & v, const int * __restrict__ u, const uint8_t * __restrict__ scales, + const int * __restrict__ v, const int * __restrict__ u, const int * __restrict__ scales, const half2 & dm2, const float * __restrict__ d8) { float sumf_d = 0.0f; @@ -373,17 +373,20 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq( #pragma unroll for (int i = 0; i < QR2_K; ++i) { - const int sc = scales[2*i]; + const int sc = scales[i/2] >> (16*(i%2)); - const int vi = (v >> (2*i)) & 0x03030303; + int sumi_d = 0; + int sumi_m = 0; +#pragma unroll + for (int j = 0; j < VDR_Q2_K_Q8_1_MMVQ; ++j) { + const int vi = (v[j] >> (2*i)) & 0x03030303; - sumf_d += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * (sc & 0xF)); // SIMD dot product + sumi_d = ggml_cuda_dp4a(vi, u[VDR_Q2_K_Q8_1_MMVQ*i + j], sumi_d); // SIMD dot product + sumi_m = ggml_cuda_dp4a(0x01010101, u[VDR_Q2_K_Q8_1_MMVQ*i + j], sumi_m); // sum of q8_1 values + } - // fill int with 4x m - int m = sc >> 4; - m |= m << 8; - m |= m << 16; - sumf_m += d8[i] * ggml_cuda_dp4a(m, u[i], 0); // multiply constant q2_K part with sum of q8_1 values + sumf_d += d8[i] * (sumi_d * (sc & 0xF)); + sumf_m += d8[i] * (sumi_m * ((sc >> 4) & 0xF)); // multiply constant q2_K part with sum of q8_1 values } const float2 dm2f = __half22float2(dm2); @@ -443,37 +446,35 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq( return sumf + d8*sumf_d8; } -#define VDR_Q3_K_Q8_1_MMVQ 1 +#define VDR_Q3_K_Q8_1_MMVQ 2 #define VDR_Q3_K_Q8_1_MMQ 2 // contiguous v/x values static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq( - const int & vl, const int & vh, const int * __restrict__ u, const uint8_t * __restrict__ scales, - const int & scale_offset, const float & d3, const float * __restrict__ d8) { + const int * __restrict__ vl, const int * __restrict__ vh, const int * __restrict__ u, const int * __restrict__ scales, + const float & d3, const float * __restrict__ d8) { float sumf = 0.0f; #pragma unroll for (int i = 0; i < QR3_K; ++i) { - const int isc = scale_offset + 2*i; - - const int isc_low = isc % (QK_K/32); - const int sc_shift_low = 4 * (isc / (QK_K/32)); - const int sc_low = (scales[isc_low] >> sc_shift_low) & 0xF; + // 4 low bits from scales[0..7], 2 high bits from scales[8..11] + const int sc_low = (scales[i/2] >> (16*(i%2))) & 0x0F; + const int sc_high = (scales[2] >> (16*(i%2) + 2*(i/2))) & 0x03; + const int sc = (sc_low | (sc_high << 4)) - 32; - const int isc_high = isc % (QK_K/64); - const int sc_shift_high = 2 * (isc / (QK_K/64)); - const int sc_high = ((scales[(QK_K/32) + isc_high] >> sc_shift_high) & 3) << 4; - - const int sc = (sc_low | sc_high) - 32; - - const int vil = (vl >> (2*i)) & 0x03030303; - - const int vih = ((vh >> i) << 2) & 0x04040404; + int sumi = 0; +#pragma unroll + for (int j = 0; j < VDR_Q3_K_Q8_1_MMVQ; ++j) { + const int vil = (vl[j] >> (2*i)) & 0x03030303; + const int vih = ((vh[j] >> i) << 2) & 0x04040404; - const int vi = __vsub4(vil, vih); + // (vil | vih) - 4 for each byte, the - 4 is done by the second dp4a + sumi = ggml_cuda_dp4a(vil | vih, u[VDR_Q3_K_Q8_1_MMVQ*i + j], sumi); // SIMD dot product + sumi = ggml_cuda_dp4a(0xFCFCFCFC, u[VDR_Q3_K_Q8_1_MMVQ*i + j], sumi); + } - sumf += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * sc); // SIMD dot product + sumf += d8[i] * (sumi * sc); } return d3 * sumf; @@ -620,12 +621,12 @@ static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq( return dm4f.x*sumf_d - dm4f.y*sumf_m; } -#define VDR_Q6_K_Q8_1_MMVQ 1 +#define VDR_Q6_K_Q8_1_MMVQ 2 #define VDR_Q6_K_Q8_1_MMQ 8 // contiguous v/x values static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq( - const int & vl, const int & vh, const int * __restrict__ u, const int8_t * __restrict__ scales, + const int * __restrict__ vl, const int * __restrict__ vh, const int * __restrict__ u, const int8_t * __restrict__ scales, const float & d, const float * __restrict__ d8) { float sumf = 0.0f; @@ -634,13 +635,18 @@ static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq( for (int i = 0; i < QR6_K; ++i) { const int sc = scales[4*i]; - const int vil = (vl >> (4*i)) & 0x0F0F0F0F; - - const int vih = ((vh >> (4*i)) << 4) & 0x30303030; + int sumi = 0; +#pragma unroll + for (int j = 0; j < VDR_Q6_K_Q8_1_MMVQ; ++j) { + const int vil = (vl[j] >> (4*i)) & 0x0F0F0F0F; + const int vih = ((vh[j] >> (4*i)) << 4) & 0x30303030; - const int vi = __vsub4((vil | vih), 0x20202020); // vi = (vil | vih) - 32 + // (vil | vih) - 32 for each byte, the - 32 is done by the second dp4a + sumi = ggml_cuda_dp4a(vil | vih, u[VDR_Q6_K_Q8_1_MMVQ*i + j], sumi); // SIMD dot product + sumi = ggml_cuda_dp4a(0xE0E0E0E0, u[VDR_Q6_K_Q8_1_MMVQ*i + j], sumi); + } - sumf += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * sc); // SIMD dot product + sumf += d8[i] * (sumi * sc); } return d*sumf; @@ -870,18 +876,31 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1( const block_q2_K * bq2_K = (const block_q2_K *) vbq + kbx; + // iqs is in 0,2..14, each 32 bytes of qs hold 4 q8_1 blocks (one per 2 bit shift) const int bq8_offset = QR2_K * (iqs / QI8_1); - const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2); + const int iqs8 = iqs % QI8_1; + + int v[VDR_Q2_K_Q8_1_MMVQ]; +#pragma unroll + for (int j = 0; j < VDR_Q2_K_Q8_1_MMVQ; ++j) { + v[j] = get_int_b4(bq2_K->qs, iqs + j); + } - const uint8_t * scales = bq2_K->scales + scale_offset; + // scale of shift i is byte 2*i + (iqs8 >= QI8_1/2) of the 8 bytes for this half + const int sc_shift = 8 * (iqs8 / (QI8_1/2)); + int scales[2]; + scales[0] = get_int_b4(bq2_K->scales, 2*(iqs / QI8_1) + 0) >> sc_shift; + scales[1] = get_int_b4(bq2_K->scales, 2*(iqs / QI8_1) + 1) >> sc_shift; - const int v = get_int_b4(bq2_K->qs, iqs); - int u[QR2_K]; + int u[VDR_Q2_K_Q8_1_MMVQ*QR2_K]; float d8[QR2_K]; #pragma unroll - for (int i = 0; i < QR2_K; ++ i) { - u[i] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs % QI8_1); + for (int i = 0; i < QR2_K; ++i) { +#pragma unroll + for (int j = 0; j < VDR_Q2_K_Q8_1_MMVQ; ++j) { + u[VDR_Q2_K_Q8_1_MMVQ*i + j] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs8 + j); + } d8[i] = __low2float(bq8_1[bq8_offset + i].ds); } @@ -893,26 +912,42 @@ static __device__ __forceinline__ float vec_dot_q3_K_q8_1( const block_q3_K * bq3_K = (const block_q3_K *) vbq + kbx; + // iqs is in 0,2..14, each 32 bytes of qs hold 4 q8_1 blocks (one per 2 bit shift) const int bq8_offset = QR3_K * (iqs / (QI3_K/2)); - const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2); + const int iqs8 = iqs % QI8_1; const float d = bq3_K->d; - const int vl = get_int_b2(bq3_K->qs, iqs); + int vl[VDR_Q3_K_Q8_1_MMVQ]; + int vh[VDR_Q3_K_Q8_1_MMVQ]; +#pragma unroll + for (int j = 0; j < VDR_Q3_K_Q8_1_MMVQ; ++j) { + vl[j] = get_int_b2(bq3_K->qs, iqs + j); + vh[j] = get_int_b2(bq3_K->hmask, iqs8 + j) >> bq8_offset; + } - // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted - const int vh = ~get_int_b2(bq3_K->hmask, iqs % (QI3_K/2)) >> bq8_offset; + // sub-block 8*(iqs/8) + 2*i + (iqs8 >= QI8_1/2) for shift i: + // 4 low bits in nibble iqs/8 of bytes, 2 high bits at bit 4*(iqs/8) + 2*(i/2) of bytes 8..11 + const int sc_shift = 8 * (iqs8 / (QI8_1/2)) + 4 * (iqs / (QI3_K/2)); + int scales[3]; +#pragma unroll + for (int l = 0; l < 3; ++l) { + scales[l] = get_int_b2(bq3_K->scales, l) >> sc_shift; + } - int u[QR3_K]; + int u[VDR_Q3_K_Q8_1_MMVQ*QR3_K]; float d8[QR3_K]; #pragma unroll for (int i = 0; i < QR3_K; ++i) { - u[i] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs % QI8_1); +#pragma unroll + for (int j = 0; j < VDR_Q3_K_Q8_1_MMVQ; ++j) { + u[VDR_Q3_K_Q8_1_MMVQ*i + j] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs8 + j); + } d8[i] = __low2float(bq8_1[bq8_offset + i].ds); } - return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, bq3_K->scales, scale_offset, d, d8); + return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, scales, d, d8); } static __device__ __forceinline__ float vec_dot_q4_K_q8_1( @@ -1025,17 +1060,25 @@ static __device__ __forceinline__ float vec_dot_q6_K_q8_1( const int scale_offset = (QI6_K/4) * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/8); const int vh_shift = 2 * ((iqs % (QI6_K/2)) / (QI6_K/4)); - const int vl = get_int_b2(bq6_K->ql, iqs); - const int vh = get_int_b2(bq6_K->qh, (QI6_K/4) * (iqs / (QI6_K/2)) + iqs % (QI6_K/4)) >> vh_shift; + int vl[VDR_Q6_K_Q8_1_MMVQ]; + int vh[VDR_Q6_K_Q8_1_MMVQ]; +#pragma unroll + for (int j = 0; j < VDR_Q6_K_Q8_1_MMVQ; ++j) { + vl[j] = get_int_b2(bq6_K->ql, iqs + j); + vh[j] = get_int_b2(bq6_K->qh, (QI6_K/4) * (iqs / (QI6_K/2)) + iqs % (QI6_K/4) + j) >> vh_shift; + } const int8_t * scales = bq6_K->scales + scale_offset; - int u[QR6_K]; + int u[VDR_Q6_K_Q8_1_MMVQ*QR6_K]; float d8[QR6_K]; #pragma unroll for (int i = 0; i < QR6_K; ++i) { - u[i] = get_int_b4(bq8_1[bq8_offset + 2*i].qs, iqs % QI8_1); +#pragma unroll + for (int j = 0; j < VDR_Q6_K_Q8_1_MMVQ; ++j) { + u[VDR_Q6_K_Q8_1_MMVQ*i + j] = get_int_b4(bq8_1[bq8_offset + 2*i].qs, iqs % QI8_1 + j); + } d8[i] = __low2float(bq8_1[bq8_offset + 2*i].ds); } From c0fd78626405c6d9b934efef0ec7291fcc20e3d6 Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Sat, 17 Oct 2026 17:20:51 +0200 Subject: [PATCH 200/431] ui : Discover & Download Models from HF Hub (#29584) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * common : resolve - download tags and list cached sidecars A Q4_0-mtp style tag now resolves the sidecar file when no model file matches it, so a solo draft or mmproj download actually pulls the file. Cached sidecar files list as their own entries so the state survives a restart, and removing such a tag deletes only the sidecar. Assisted-by: pi:zai-org/GLM-5.3 * ui : optional sanitized raw HTML in markdown Add an allowHtml prop to MarkdownContent: raw HTML found in the markdown is rendered after DOMPurify sanitization instead of being escaped as literal text. Default stays escaped. Assisted-by: pi:GLM-5.3-Flash * ui : add the discover models components The Discover surface: the curated-catalog list with its search and skeletons, the details pane with its readme, metadata, Hub stats and download options, and the standalone download progress bar that replaces the plain one. The quant download button sizes its select with a new xs trigger, so the select gains that size and the muted box look. The store stays incomplete when a repo fails, so the next mount retries it. Assisted-by: pi:llama.cpp/DeepSeek-V4.1-Flash * ui : put the discover view in the models manager The manager dialog gains its second view: the Discover Models call to action fades the table out, slides the title with its compass mark, and fades the discover surface in; the arrow button returns to the table. Assisted-by: pi:llama.cpp/DeepSeek-V4.1-Flash # Conflicts: # tools/ui/src/lib/components/ui/button/button.svelte * ui : add the Enable Discover Models setting The flag gates browsing and downloading HuggingFace GGUF models, so it is declared on the discover layer; the repo-tree lookup in the manager layer above reads it from here. Assisted-by: pi:llama.cpp/DeepSeek-V4.1-Flash * ui : resume model downloads from the pause point A stopped download keeps its partial file instead of deleting it, so re-posting the tag resumes from disk. The first progress record now carries the resume offset, the paused snapshot survives into the resumed entry, and a pause settles as soon as it is requested. Assisted-by: pi * ui : lead the discover list with downloads The discover sidebar lists running downloads above the suggested models, and steps aside while a query is active. A paused download stays in the manager table, where it is resumed or dropped. Assisted-by: pi * ui : make resuming a paused download work Re-posting the tag of a paused download hit the router's "already exists" check, because a paused download parks its entry as DOWNLOADED until the next reload: a download entry that is not running is now replaced and resumed. The UI keeps the paused row when the request still fails, instead of dropping it and looking finished. Assisted-by: pi * ui : remember unfinished model downloads A partial file is named after its content hash, so nothing on disk says which model it belongs to once the router restarts. The router now keeps the tags of the downloads it started, clears a tag when the download finishes or is deleted, and lists what is left as still downloading without starting anything. The UI adopts those tags as paused on every list fetch, so a refresh or a fresh open shows an incomplete download and can resume it from the files already on disk. Assisted-by: pi * ui : show a paused download's state on its chip A paused quant chip reads as paused at rest, the way a running one shows its spinner, and offers the resume action on hover. The selector's download row gets a View in Discover action beside its cancel. Assisted-by: pi * ui : collapse the download options by default The copyable download command starts collapsed so the option rows lead; expanding it stays one click away. Assisted-by: pi * ui : keep decision models out of the hub lists Decision models answer with calibrated probabilities instead of text, so llama-ui cannot run them: the hub data layer drops them from the catalog and every search, keyed by the decision-model tag or a text-classification pipeline. Assisted-by: pi * ui : read download tags as quant and sidecar A `:-` tag now parses into its quant and sidecar instead of one opaque quant token, and ModelsService.draftSidecarsByRepo collects the router listing's sidecar-only entries per repo, replacing the inline copy in the models store. Assisted-by: pi * ui : cluster draft downloads under their main model Tracked downloads group by repo: the main weights lead and the draft sidecars ride inside their item as compact sub-lines, on the selector, the discover list and the manager table. The main model's controls pause, resume and cancel the whole cluster, and the manager's download rows select into the configuration pane. Assisted-by: pi * ui : act on tracked downloads from the table rows The table passes its clusters' draft sub-lines to the model rows. A row's actions cascade pause and resume, an in-flight row offers Cancel download while a paused one keeps Delete from disk, and the confirm dialog's copy follows the state. Assisted-by: pi * ui : give downloads a configuration pane of their own Selecting a tracked download opens the pane in a download mode: live progress with pause, resume and cancel, and See in Discover, instead of metadata and load actions that a model without files cannot offer. Assisted-by: pi * ui : make the discover details self-fetching and responsive ModelsDiscoverDetailsContainer owns the details, tree and readme fetches so hosts outside the discover view can render the pane, and the details layout adapts to narrow panes with container queries instead of viewport breakpoints. Assisted-by: pi * ui : download a matching draft with the main model A main quant download now pulls the quant-matched draft of the most preferred kind the repo ships (DFlash before MTP), fire-and-forget, so speculative decoding is ready when the main lands. Manual draft chips stay available; the options row also collapses to one column in narrow panes. Assisted-by: pi * ui : keep discover off by default and couple it with the hub setting Discover Models starts unchecked. Enabling it turns the Hugging Face Hub setting on and disabling it turns it back off, so the checkbox never lies; with discover off the hub setting stays available on its own for the display data. Stored and reset state are corrected the same way on load. Assisted-by: pi * ui : open the manage models dialog on the manager view The dialog now opens on the manager instead of returning to the last view, discover entries still land on the discover view while the setting allows it, and the discover surfaces follow the setting's state. Assisted-by: pi * ui : give the discover search room for its focus ring The sticky search bar insets from the panel edges so the focus ring stays inside the scroll container, and rows scroll behind an opaque bar instead of through it. Assisted-by: pi * common: settle sidecar names on the listing and drop unfinished partials An infix or trailing sidecar token is ambiguous: draft heads use it (Model-MTP-BF16.gguf), so do full models with embedded heads (Model-Q8D-MTP.gguf). Such a name is a sidecar only next to token-free main weights of the same family, while mmproj or imatrix anywhere in the name, or a leading token, decides alone (mmproj_f16.gguf). A quant tag always carries its bit count, so a sidecar named after no quant (mtp-Model-it.gguf) gets the bare tag: the listing, the plan and remove all use it. An imatrix tag resolves into a plan slot of its own, downloaded but never wired into the params. remove resolves the tag the way a download does and deletes the partial files of every planned blob, so cancelling a running download or dropping one left unfinished by an earlier run leaves nothing on disk. The router passes its token and releases its lock around the call, and records the unfinished downloads with an atomic write. llama download succeeds on a sidecar-only plan and prints its paths. * ui : move dialog views and draft states into enums and constants The models dialog's views become the ModelsDialogView enum, the draft subline's states join ModelRowDownloadState, the view fade and the copy-feedback reset move to ui constants, and the dialog's compass marks come from DISCOVER_ICON. Assisted-by: pi * ui : drop the unused discover surface The dead openDiscoverModels twin, the commented-out command divider, the details pane's unused onClose and its header's close button, and a redundant type cast leave; the command's copy feedback joins the shared reset constant. Assisted-by: pi * ui : trim the download option comments and highlight the chat template The comment verbosity the options panel and its chips carried shrinks to the non-obvious why, and the chat template dialog renders through SyntaxHighlightedCode as jinja instead of a plain pre block. Assisted-by: pi * common: only a .gguf file is a sidecar, and mmproj/imatrix as whole words A non-GGUF file named like a sidecar (quantization-mmproj-Q8_0.json) is picked as the mmproj, and a sidecar word inside a longer word of a model name (Phi-2-OmniMatrix.i1-Q4_K_M.gguf) drops the model. * ui: dedupe draft sidecar badges per quant The draft sidecar badges deduped by kind and repo without the quant, so with a draft cached at two quants only one badge survived, and a delete of the other main quant found no badge to remove its sidecar with. Also restores the getDownloadEntries doc lost when the clusters method landed above it. Assisted-by: pi (zai-org/GLM-5.3-Flash) * ui: adopt the incomplete download status and drop the paused-ids registry The store inferred an unfinished download from a downloading entry it held no local state for, which misread a download running in another tab as paused and offered a resume the server rejects. The listing carries the "incomplete" status now, so the adoption keys off it directly. The paused ids no longer need to survive in localStorage: the server remembers them across restarts, and the progress snapshot was stale after a reload anyway. The stale localStorage key is removed. Assisted-by: pi (zai-org/GLM-5.3-Flash) * ui: quant-match the pause and cancel cascades Pause, resume and cancel cascaded to every draft sidecar of the repo while delete only took the quant-matched ones, so canceling a Q4_0 download also killed a Q8_0 draft that belongs to another quant's main. Every cascade acts on the quant-matched drafts now, matching what a delete takes with it. Assisted-by: pi (zai-org/GLM-5.3-Flash) * ui: collapse the duplicated draft subline markup The downloaded and in-flight branches repeated the identical connector, label and badge markup; one block with a conditional tail keeps the two states in sync. Assisted-by: pi (zai-org/GLM-5.3-Flash) * ui: scrub positioned styles out of sanitized model card html The style attribute stays allowed for the KaTeX output, which let a third party card draw a fixed overlay over the app. The sanitize hook drops position:fixed and position:absolute values and leaves the KaTeX layout styles alone. Assisted-by: pi (zai-org/GLM-5.3-Flash) * ui: run the discover mount fetch once and reuse the shared repoOf helper fetch() toggles its loading flag synchronously inside the mount effect, which re-ran the fetch on every toggle and retried failed catalog repos endlessly; the effect now untracks the call. The local repoOf copy and the duplicate services import give way to the shared helpers. Assisted-by: pi (zai-org/GLM-5.3-Flash) * ui: narrow the spec-type map to the draft sidecars SPEC_TYPE carried empty entries for the aux sidecars it is never indexed with; the map and the command component now type the badge as a draft sidecar through a draftBadgeOf helper instead of a cast. Assisted-by: pi (zai-org/GLM-5.3-Flash) * ui: name the decision-model tags and markdown cache keys The decision-model hub tag and pipeline tag move into the huggingface constants next to the other pipeline tags, and the markdown transform cache prefixes get names instead of bare one-letter tokens. Assisted-by: pi (zai-org/GLM-5.3-Flash) * ui: read the discover setting through one truthiness check Three places read the setting with !== false and one with Boolean(); all four read it the same way now. Assisted-by: pi (zai-org/GLM-5.3-Flash) * ui: compare the models dialog view against its enum The template compared the view state against the 'manage' literal in three spots while the script used the enum. Assisted-by: pi (zai-org/GLM-5.3-Flash) * nit: Remove `+` before draft sidecar badge * server: list unfinished downloads as incomplete A download that never finished was reported as "downloading", the same status a live download carries, so a client could not tell a running download from a dead one. The tags remembered in unfinished-downloads now list under a dedicated "incomplete" status, without an instance. Assisted-by: pi (zai-org/GLM-5.3-Flash) * server: download a model with its sidecars as -hf does Resolution goes back to master. POST /models takes the sidecar kinds to pull with the model (mmproj by default) and the download child gets them as -hf flags, so the server picks the files. /models lists real models only, each with the sidecars it has on disk, and incomplete downloads keep their sidecars across restarts. A stopped download keeps its partial files for the next attempt, and DELETE purges them through the same plan the download follows. * ui: download sidecars as parameters of their model The download options keep their chips, but a chip no longer posts a sidecar tag: the store maps it to its model and the sidecar kinds to pull, the quant chip takes its companion draft in the same download, and draft badges come from the sidecars each cached model lists. The server adds the sidecars a cached model lacks when it is posted again, and DELETE removes a single sidecar of a cached model. * ui : mirror the server's sidecar lookup in draft badges The server resolves a sidecar by the case-sensitive `-` token in the same directory as the weights. The UI matched the token loosely, so a badge could offer a draft the server never pulls. Assisted-by: DeepSeek-V4.1-Flash * ui : post the model's sidecar kinds with its download The store keeps one download per model: the chip's kinds ride along with the model's own sidecars, and a download row no longer expands into one row per sidecar. Draft badges carry no quant of their own yet, so a sidecar of a cached model is never labelled with the model's quant. Assisted-by: DeepSeek-V4.1-Flash * ui : join a quant's draft into its download chip One chip per quant, its draft badged inside it and its size covering both files. The draft is the file the server would pick: the same directory, the exact quant, else the closest bit depth, so a 4-bit model can pair with a 16-bit head. Assisted-by: DeepSeek-V4.1-Flash * ui : drop the draft download rows from the manager and selector A download is one row per model again: the drafts ride with it, and the rows read the sidecars on disk off the listing instead of pairing sidecar entries with their model. Pause, resume and cancel act on the model's own download. Assisted-by: DeepSeek-V4.1-Flash * ui : keep the discover download list one row per model The list leads with one row per downloading model, its sidecars as sub-lines. A row badges only the draft kinds the server can resolve, so a badge never outlives the file its lookup would pick. Assisted-by: DeepSeek-V4.1-Flash * ui : keep the discover chips readable in dark mode The metadata chips, the chat template button and the View on Hugging Face link carried a faint border and no surface over the dialog background, so they were near invisible. They take the tertiary button surface now, and their icons size with size-* so the variant's base rule leaves them alone. Assisted-by: DeepSeek-V4.1-Flash * ui : carry the draft through the download options One chip per main quant, its draft badged on it and the chip's downloaded state covering both files. The drafts sit behind an opt-in checkbox with an inline type pick, and each row opens its own command: it pins the row's quant and only names the draft type, so -hfd is gone. Assisted-by: DeepSeek-V4.1-Flash * ui : line the details skeleton up with the loaded pane Same root padding and container context as the loaded view, so the container queries resolve alike. The download options card pads its blocks instead of itself, the readme gets its card, and a description line stands in for the catalog paragraph. Assisted-by: DeepSeek-V4.1-Flash * common : drop the partials of a removed sidecar Removing a sidecar alone left the .downloadInProgress files of the sidecars a download had planned in blobs/. The removal purges them through the same plan the download follows, and the test covers a finished draft beside an unfinished one, whose partial goes while the model keeps its files. Assisted-by: DeepSeek-V4.1-Flash * server : name why a download is refused A model that is already downloading, or that is running, answered "already exists". It now says which: "is already downloading", or "is running, unload it first", both before the resumable check. The router test covers the two. Assisted-by: DeepSeek-V4.1-Flash * ui : post a download once per click Two quick clicks on the same chip posted twice, and the second answer was "already exists". The store tracks the model as pending, returns early while it is pending or in flight, and reads it as in flight before the server answers, so the chip shows the download at once. The new client test pins the flows: one POST per double click, the chip reading downloaded once the list reports the model, a resume keeping its sidecars, a cancel dropping the whole download, and a reload restoring the paused state. Assisted-by: DeepSeek-V4.1-Flash * ui : keep a plain download row the height of the others The sub-lines wrapper of a download row was rendered even with nothing in it, so its padding made the row taller than a plain one and the overlaid progress bar sat in that empty strip. It renders only with sub-lines now, in the selector and in the discover list, and the selector row takes the phone padding its neighbours have. Assisted-by: DeepSeek-V4.1-Flash * ui : badge a repo's draft once The args of a loaded model and the listing of a cached one report the same draft in different words, one with the file's quant and one with the kind alone, so the merge kept both and a repo row showed the same kind twice. The badges merge per kind now, the args one winning as it knows the quant, and the test pins both sources of a real repo. Assisted-by: DeepSeek-V4.1-Flash * server : report the sidecars of a preset or a download on its way A model the preset also names is registered as a preset, so its drafts were never reported and no row could badge them. A download on its way reported none either, though the record of the kinds it pulls is right there: those win over the plan now, the plan resolves through the preset's hf-repo, and a models dir stays out of it. Assisted-by: DeepSeek-V4.1-Flash * ui : badge a download's drafts and open its model from the row A draft still coming down badges the model id now, the way a landed one does, while an on-disk draft keeps its checked sub-line. The row itself opens the model, the status slot beside it pauses or resumes, and the manager resolves a focus on a download row so its panel opens. Assisted-by: DeepSeek-V4.1-Flash * common : report a removed sidecar that only had a partial Removing a sidecar alone dropped the result of the partial purge, so a sidecar that only ever landed as a .downloadInProgress blob answered false and the UI had no reason to refetch. The purge result is folded into the return now, and the test covers a partial-only removal. Assisted-by: DeepSeek-V4.1-Flash * ui : ignore a pause while the download POST is still on its way The chip reads as downloading before the server answers, so a quick second click unloaded a model that had not started and left the store neither in flight nor paused. A pause returns early while the POST is pending, before it touches the feed or the paused marker. The client test holds the POST open and pins it. Assisted-by: DeepSeek-V4.1-Flash * ui : offer a missing draft as its own chip A model already on disk without its draft had no way to add it: the quant chip carried the draft, so it read as a download of both. The quant chip keeps to its own state now, its joined draft shows only while it rides along or is on disk, and a downloaded quant missing its draft gets a chip of its own - badge, size, download - that posts the model with that kind, so the server keeps the files already there and pulls the draft. Assisted-by: DeepSeek-V4.1-Flash * server : list a paused sidecar download by the files on disk A model on disk whose sidecar download stops keeps its unfinished record, and the listing reported the kinds that record plans as if they were on disk: the UI checked a draft that never landed and offered no way to add it. The planned kinds now stand in only while the download runs. * chore: Clean up comments * server : list sidecars only for a model that resolves from a HF repo A preset on a local file has no HF repo, yet the listing resolved its name as one: the plan threw on the invalid repo format and GET /models failed for every model. The sidecars now come from the repo a preset names or a cached model is listed under, and a local file lists without them. * common : fix download cancel regression Signed-off-by: Adrien Gallouët * common : preserve model partials when removing sidecars Signed-off-by: Adrien Gallouët * server : one sidecar table and an unfinished downloads class Address review from @ggerganov. The sidecar kinds live in one table in common/download, mapping each name to its download option and plan file; the server validates, builds its options and lists cached sidecars through it instead of its own lists. The unfinished downloads move into server_unfinished_downloads, a record with its own mutex and a struct per entry, and server_models exposes them through unfinished_downloads_get and unfinished_downloads_remove. * common: one sidecar table, and removals that keep shared files The resolution, the cached listing and the removal follow a single table of sidecar kinds, so a kind is declared once with its file keyword, whether it is a draft, its option and its plan file. The server reads the same table for the kinds it accepts, the default sidecars and the child flags. Removing a tag keeps the files another cached tag still loads, such as an mmproj named after the removed quant. A sidecar removed on its own goes for every tag that resolves it. An unfinished download records the exact paths of its partials when it starts. A removal deletes those without asking the hub, and keeps a partial another recorded download also writes. The record is taken once the child is spawned, and the files go under the models lock, so a download of the same tag cannot start while they are removed. * server: drop stale partials and refuse a tag that resolves nothing Recording a download again deletes the partials of the earlier record that the new plan no longer writes, such as those of an older hub revision or of a sidecar the new request leaves out. A posted tag that resolves no file of the repo is refused before the download starts. Removing a model follows a single path, whether the reload erased its entry during the wait or not. * server: fix the CI race between the router download tests The pytest workers run in parallel on one shared cache, so a download test could find the file another test had just fetched and see no progress, or start from the unfinished downloads record another test wrote. Each download test now runs on an empty cache of its own. * server: refuse a DELETE of an unknown name and decide downloads in load() A DELETE of a name that is neither a model nor a recorded download went to common_download_remove, and without a tag it wiped the whole repo cache, including the files of a loaded quant or of a download on its way. Such a name is now not found, as it was before. A POST that downloads over an existing entry checked its state outside the lock, so a load of the same model in between got its instance killed. The checks now run in load(), under the lock, right before the spawn. The fake load time of the tests applies to model loads only, since a download worker loads none. * server: add incomplete_sidecars to the /models API A sidecar added to a cached model and then paused kept its record, but the listing showed only the complete model, so a fresh UI lost the pause. The model entry now carries incomplete_sidecars, the kinds its unfinished download has not written yet; a POST with the kind resumes one and a DELETE with it drops its partial. * common : drop a kept partial when a resume gets 416 Now that partials survive between attempts, an old or complete partial makes every retry fail with the same 416. Delete it so the next attempt can start over from zero. Signed-off-by: Adrien Gallouët * common: drop a kept partial when a resume gets 200 A server that ignores the range answers 200 to a resume, and the kept partial then fails every attempt the same way a 416 does. Drop it as well, so the next attempt starts over without a range. * server: refuse a download that shares a partial with one on its way Two downloads of quants that share a file, the mmproj or an mtp picked by the closest bits, appended to the same partial and renamed a file with interleaved bytes. A download whose plan writes a partial that a running download also writes is now refused until that one is done. * common : check the saved etag before resuming a partial download Before resuming, compare the etag saved with the partial to the one the server now serves. If it is missing or different, start over, so a resumed download never mixes bytes from two versions of the file. Signed-off-by: Adrien Gallouët --------- Signed-off-by: Adrien Gallouët Co-authored-by: Pascal Co-authored-by: Adrien Gallouët --- common/download.cpp | 236 ++++++--- common/download.h | 39 +- tests/test-model-resolution.cpp | 122 +++++ tools/server/README.md | 37 +- tools/server/server-models.cpp | 421 +++++++++++++++-- tools/server/server-models.h | 65 +++ tools/server/tests/unit/test_router.py | 237 ++++++++++ tools/server/tests/utils.py | 5 +- .../MarkdownContent/MarkdownContent.svelte | 34 +- .../MarkdownContent/markdown-processor.ts | 31 +- .../MarkdownContent/safe-html-config.ts | 138 ++++++ .../app/dialogs/DialogManageModels.svelte | 181 ++++++- .../app/models/ModelDraftSidecars.svelte | 2 - .../app/models/ModelDraftSubline.svelte | 53 +++ .../lib/components/app/models/ModelId.svelte | 15 + .../models/ModelsManager/ModelsManager.svelte | 32 +- .../ModelsManagerDownloadControl.svelte | 5 +- .../ModelsManagerModelConfiguration.svelte | 98 +++- ...elsManagerModelConfigurationActions.svelte | 143 ++++-- ...delsManagerModelConfigurationHeader.svelte | 31 +- .../ModelsManagerModelRow.svelte | 96 ++-- .../ModelsManagerModelsTable.svelte | 8 +- .../app/models/ModelsManager/row-actions.ts | 31 +- .../app/models/ModelsManager/utils.ts | 31 +- .../ModelsSelectorDownloadItem.svelte | 160 +++++-- .../ModelsSelector/ModelsSelectorList.svelte | 9 +- .../app/models/discover/ModelsDiscover.svelte | 210 ++++++++ .../ModelsDiscoverChatTemplateDialog.svelte | 75 +++ .../ModelsDiscoverDetails.svelte | 96 ++++ .../ModelsDiscoverDetailsContainer.svelte | 66 +++ ...odelsDiscoverDetailsDownloadOptions.svelte | 258 ++++++++++ ...tailsDownloadOptionsDownloadCommand.svelte | 131 +++++ ...sDownloadOptionsDraftDownloadButton.svelte | 61 +++ ...sDownloadOptionsQuantDownloadButton.svelte | 272 +++++++++++ ...lsDiscoverDetailsDownloadOptionsRow.svelte | 134 ++++++ .../download-options.utils.ts | 57 +++ .../index.ts | 24 + .../ModelsDiscoverDetailsHeader.svelte | 96 ++++ .../ModelsDiscoverDetailsHfHubStats.svelte | 31 ++ .../ModelsDiscoverDetailsMetadata.svelte | 82 ++++ .../ModelsDiscoverDetailsMetadataItem.svelte | 18 + .../ModelsDiscoverDetailsReadme.svelte | 17 + .../ModelsDiscoverDetailsSkeleton.svelte | 98 ++++ .../discover/ModelsDiscoverDetails/index.ts | 37 ++ .../ModelsDiscoverList.svelte | 41 ++ .../ModelsDiscoverListItem.svelte | 256 ++++++++++ .../ModelsDiscoverListItemSkeleton.svelte | 39 ++ .../ModelsDiscoverListSearch.svelte | 30 ++ .../discover/ModelsDiscoverList/index.ts | 16 + .../components/app/models/discover/index.ts | 16 + .../ui/src/lib/components/app/models/index.ts | 3 + .../settings/SettingsChat/SettingsChat.svelte | 7 + .../SettingsChat/SettingsChatFields.svelte | 14 +- .../ui/select/select-content.svelte | 4 +- .../ui/select/select-trigger.svelte | 14 +- .../lib/constants/huggingface.constants.ts | 6 + tools/ui/src/lib/constants/icons.constants.ts | 4 + tools/ui/src/lib/constants/index.ts | 1 + .../models-discover-download.constants.ts | 41 ++ .../constants/models-discover.constants.ts | 6 + .../lib/constants/settings-keys.constants.ts | 1 + .../src/lib/constants/settings.constants.ts | 13 +- .../ui/src/lib/constants/storage.constants.ts | 2 - tools/ui/src/lib/constants/ui.constants.ts | 6 + tools/ui/src/lib/enums/index.ts | 9 +- tools/ui/src/lib/enums/model.enums.ts | 16 +- tools/ui/src/lib/enums/server.enums.ts | 2 + tools/ui/src/lib/enums/ui.enums.ts | 6 + .../src/lib/services/huggingface.service.ts | 98 ++-- tools/ui/src/lib/services/models.service.ts | 106 +++-- tools/ui/src/lib/stores/index.ts | 3 + .../stores/models-discover/index.svelte.ts | 267 +++++++++++ .../ui/src/lib/stores/models/index.svelte.ts | 72 +-- .../ui/src/lib/stores/models/status.svelte.ts | 447 ++++++++++++------ .../src/lib/stores/settings/index.svelte.ts | 34 +- tools/ui/src/lib/stores/ui.svelte.ts | 10 + tools/ui/src/lib/types/api.d.ts | 5 + tools/ui/src/lib/types/huggingface.d.ts | 2 + tools/ui/src/lib/types/index.ts | 11 + tools/ui/src/lib/types/models-discover.d.ts | 56 +++ tools/ui/src/lib/types/models.d.ts | 2 + tools/ui/src/lib/utils/index.ts | 9 +- tools/ui/src/lib/utils/sidecars.ts | 18 +- .../markdown-content-sanitize.svelte.test.ts | 160 +++++++ .../models-download-flows.svelte.test.ts | 155 ++++++ .../models-manager-utils.svelte.test.ts | 28 +- .../models-sidecar-badges.svelte.test.ts | 99 ++++ tools/ui/tests/unit/hf-helpers.test.ts | 37 ++ tools/ui/tests/unit/model-id-parser.test.ts | 59 +++ .../tests/unit/model-sidecar-grammar.test.ts | 14 +- 90 files changed, 5688 insertions(+), 580 deletions(-) create mode 100644 tools/ui/src/lib/components/app/content/MarkdownContent/safe-html-config.ts create mode 100644 tools/ui/src/lib/components/app/models/ModelDraftSubline.svelte create mode 100644 tools/ui/src/lib/components/app/models/discover/ModelsDiscover.svelte create mode 100644 tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverChatTemplateDialog.svelte create mode 100644 tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetails.svelte create mode 100644 tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsContainer.svelte create mode 100644 tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptions.svelte create mode 100644 tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsDownloadCommand.svelte create mode 100644 tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsDraftDownloadButton.svelte create mode 100644 tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsQuantDownloadButton.svelte create mode 100644 tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsRow.svelte create mode 100644 tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/download-options.utils.ts create mode 100644 tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/index.ts create mode 100644 tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsHeader.svelte create mode 100644 tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsHfHubStats.svelte create mode 100644 tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsMetadata.svelte create mode 100644 tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsMetadataItem.svelte create mode 100644 tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsReadme.svelte create mode 100644 tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsSkeleton.svelte create mode 100644 tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/index.ts create mode 100644 tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverList.svelte create mode 100644 tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverListItem.svelte create mode 100644 tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverListItemSkeleton.svelte create mode 100644 tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverListSearch.svelte create mode 100644 tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/index.ts create mode 100644 tools/ui/src/lib/components/app/models/discover/index.ts create mode 100644 tools/ui/src/lib/constants/models-discover-download.constants.ts create mode 100644 tools/ui/src/lib/stores/models-discover/index.svelte.ts create mode 100644 tools/ui/src/lib/types/models-discover.d.ts create mode 100644 tools/ui/tests/client/markdown-content-sanitize.svelte.test.ts create mode 100644 tools/ui/tests/client/models-download-flows.svelte.test.ts create mode 100644 tools/ui/tests/client/models-sidecar-badges.svelte.test.ts diff --git a/common/download.cpp b/common/download.cpp index ae0f7ffede42..38f26e1b497a 100644 --- a/common/download.cpp +++ b/common/download.cpp @@ -39,6 +39,9 @@ // downloader // +// the suffix of the file a download writes until it completes +static const std::string PARTIAL_SUFFIX = ".downloadInProgress"; + static void write_etag(const std::string & path, const std::string & etag) { const std::string etag_path = path + ".etag"; fs_write_atomic(std::filesystem::u8path(etag_path), etag); @@ -186,6 +189,13 @@ static bool common_pull_file(httplib::Client & cli, [&](const httplib::Response &response) { if (p.downloaded > 0 && response.status != 206) { LOG_WRN("%s: server did not respond with 206 Partial Content for a resume request. Status: %d\n", func, response.status); + if (response.status == 416 || response.status == 200) { + // the kept partial cannot be resumed: 416 for a complete or stale one, + // 200 from a server that ignores the range; drop it, so the next + // attempt starts over instead of wedging on the same answer + ofs.close(); + remove(path_tmp.c_str()); + } return false; } if (p.downloaded == 0 && response.status != 200) { @@ -214,7 +224,7 @@ static bool common_pull_file(httplib::Client & cli, if (progress_step >= p.total / 1000 || p.downloaded == p.total) { if (callback) { callback->on_update(p); - if (callback->is_cancelled()) { + if (callback->is_cancelled() && (p.total == 0 || p.downloaded != p.total)) { return false; } } @@ -331,10 +341,24 @@ static int common_download_file_single_online(const std::string & url, } bool success = false; - const std::string path_temporary = path + ".downloadInProgress"; + const std::string path_temporary = path + PARTIAL_SUFFIX; int delay = retry_delay_seconds; + if (!skip_etag && !etag.empty()) { + if (std::filesystem::exists(path_temporary) && read_etag(path_temporary) != etag) { + LOG_WRN("%s: partial file does not match the server etag, discarding it: %s\n", __func__, path_temporary.c_str()); + if (remove(path_temporary.c_str()) != 0) { + LOG_ERR("%s: unable to delete file: %s\n", __func__, path_temporary.c_str()); + return -1; + } + } + write_etag(path_temporary, etag); + } + if (opts.callback) { + // report the resume offset, so a resumed file does not start at 0 + p.downloaded = supports_ranges && std::filesystem::exists(path_temporary) ? + std::filesystem::file_size(path_temporary) : 0; opts.callback->on_start(p); } @@ -372,6 +396,7 @@ static int common_download_file_single_online(const std::string & url, } if (!etag.empty() && !skip_etag) { write_etag(path, etag); + remove((path_temporary + ".etag").c_str()); } success = true; break; @@ -381,12 +406,7 @@ static int common_download_file_single_online(const std::string & url, if (opts.callback) { opts.callback->on_done(p, success); } - if (opts.callback && opts.callback->is_cancelled() && - std::filesystem::exists(path_temporary)) { - if (remove(path_temporary.c_str()) != 0) { - LOG_ERR("%s: unable to delete temporary file: %s\n", __func__, path_temporary.c_str()); - } - } + // a stopped download keeps path_temporary, so the next attempt resumes from it if (!success) { LOG_ERR("%s: download failed after %d attempts\n", __func__, max_attempts); return -1; // max attempts reached @@ -594,33 +614,65 @@ static hf_cache::hf_file find_best_sibling(const hf_cache::hf_files & files, return best; } -static hf_cache::hf_file find_best_mmproj(const hf_cache::hf_files & files, - const std::string & model) { - return find_best_sibling(files, model, "mmproj"); +static const std::vector sidecars = { + { "mmproj", "mmproj", false, &common_download_opts::download_mmproj, &common_download_hf_plan::mmproj }, + { "mtp", "mtp-", true, &common_download_opts::download_mtp, &common_download_hf_plan::mtp }, + { "dflash", "dflash-", true, &common_download_opts::download_dflash, &common_download_hf_plan::dflash }, + { "eagle3", "eagle3-", true, &common_download_opts::download_eagle3, &common_download_hf_plan::eagle3 }, + { "dspark", "dspark-", true, &common_download_opts::download_dspark, &common_download_hf_plan::dspark }, +}; + +const std::vector & common_download_sidecars() { + return sidecars; } -static hf_cache::hf_file find_best_mtp(const hf_cache::hf_files & files, - const std::string & model, - const std::string & tag = "") { - return find_best_sibling(files, model, "mtp-", tag); +bool common_download_opts_add_sidecar(common_download_opts & opts, const std::string & kind) { + for (const auto & s : sidecars) { + if (kind == s.name) { + opts.*s.opt = true; + return true; + } + } + return false; } -static hf_cache::hf_file find_best_eagle3(const hf_cache::hf_files & files, - const std::string & model, - const std::string & tag = "") { - return find_best_sibling(files, model, "eagle3-", tag); +std::vector common_download_plan_sidecars(const common_download_hf_plan & plan) { + std::vector out; + for (const auto & s : sidecars) { + if (!(plan.*s.file).path.empty()) { + out.emplace_back(s.name); + } + } + return out; } -static hf_cache::hf_file find_best_dflash(const hf_cache::hf_files & files, - const std::string & model, - const std::string & tag = "") { - return find_best_sibling(files, model, "dflash-", tag); +std::vector common_download_plan_partials(const common_download_hf_plan & plan) { + std::vector out; + + // a file in the cache resolves to its snapshot entry, a file still to download to its blob + auto add = [&](const std::string & kind, const hf_cache::hf_file & f) { + if (!f.local_path.empty() && f.local_path != f.final_path) { + out.push_back({ kind, f.local_path + PARTIAL_SUFFIX }); + } + }; + + for (const auto & f : plan.model_files) { + add("", f); + } + for (const auto & s : sidecars) { + add(s.name, plan.*s.file); + } + return out; } -static hf_cache::hf_file find_best_dspark(const hf_cache::hf_files & files, - const std::string & model, - const std::string & tag = "") { - return find_best_sibling(files, model, "dspark-", tag); +// true when `name` carries the keyword of a sidecar kind +static bool name_is_sidecar(const std::string & name) { + for (const auto & s : sidecars) { + if (name.find(s.keyword) != std::string::npos) { + return true; + } + } + return false; } static bool gguf_filename_is_model(const std::string & filepath) { @@ -633,12 +685,7 @@ static bool gguf_filename_is_model(const std::string & filepath) { filename = filename.substr(pos + 1); } - return filename.find("mmproj") == std::string::npos && - filename.find("imatrix") == std::string::npos && - filename.find("mtp-") == std::string::npos && - filename.find("eagle3-") == std::string::npos && - filename.find("dflash-") == std::string::npos && - filename.find("dspark-") == std::string::npos; + return filename.find("imatrix") == std::string::npos && !name_is_sidecar(filename); } static hf_cache::hf_file find_best_model(const hf_cache::hf_files & files, @@ -730,8 +777,11 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model & } } else { primary = find_best_model(all, tag); - // a requested sidecar can resolve on its own, without a full model of the same tag - if (primary.path.empty() && !opts.download_mtp && !opts.download_dflash && !opts.download_eagle3 && !opts.download_dspark) { + // a requested draft can resolve on its own, without a full model of the same tag + const bool want_draft = std::any_of(sidecars.begin(), sidecars.end(), [&](const common_download_sidecar & s) { + return s.draft && opts.*s.opt; + }); + if (primary.path.empty() && !want_draft) { LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str()); list_available_gguf_files(all); return plan; @@ -743,24 +793,22 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model & plan.model_files = get_split_files(all, primary); } - if (opts.download_mmproj && !primary.path.empty()) { - plan.mmproj = find_best_mmproj(all, primary.path); - } - if (opts.download_mtp) { - plan.mtp = find_best_mtp(all, primary.path, tag); - } - if (opts.download_dflash) { - plan.dflash = find_best_dflash(all, primary.path, tag); - } - if (opts.download_eagle3) { - plan.eagle3 = find_best_eagle3(all, primary.path, tag); - } - if (opts.download_dspark) { - plan.dspark = find_best_dspark(all, primary.path, tag); + // a draft resolves at the tag, the mmproj pairs with the model + for (const auto & s : sidecars) { + if (!(opts.*s.opt)) { + continue; + } + if (s.draft) { + plan.*s.file = find_best_sibling(all, primary.path, s.keyword, tag); + } else if (!primary.path.empty()) { + plan.*s.file = find_best_sibling(all, primary.path, s.keyword); + } } - if (primary.path.empty() && - plan.mtp.local_path.empty() && plan.dflash.local_path.empty() && plan.eagle3.local_path.empty() && plan.dspark.local_path.empty()) { + const bool has_draft = std::any_of(sidecars.begin(), sidecars.end(), [&](const common_download_sidecar & s) { + return s.draft && !(plan.*s.file).local_path.empty(); + }); + if (primary.path.empty() && !has_draft) { LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str()); list_available_gguf_files(all); } @@ -926,6 +974,11 @@ std::string common_docker_resolve_model(const std::string & docker) { } } +// a cached file that lists a model under its tag: the first split of a model, not a sidecar +static bool is_listed_model(const gguf_split_info & split) { + return split.index == 1 && !split.tag.empty() && !name_is_sidecar(split.prefix); +} + std::vector common_list_cached_models() { std::unordered_set seen; std::vector result; @@ -934,12 +987,7 @@ std::vector common_list_cached_models() { for (const auto & f : files) { auto split = get_gguf_split_info(f.path); - if (split.index != 1 || split.tag.empty() || - split.prefix.find("mmproj") != std::string::npos || - split.prefix.find("mtp-") != std::string::npos || - split.prefix.find("eagle3-") != std::string::npos || - split.prefix.find("dflash-") != std::string::npos || - split.prefix.find("dspark-") != std::string::npos) { + if (!is_listed_model(split)) { continue; } if (seen.insert(f.repo_id + ":" + split.tag).second) { @@ -970,6 +1018,44 @@ std::string common_download_resolve_path(const std::string & hf_repo_with_tag, c return find_best_model(files, tag).local_path; } +// the files -hf loads from the cache, the model with every sidecar, for each cached +// tag of `repo_id` other than `tag` +static std::unordered_set files_in_use(const std::string & repo_id, const std::string & tag) { + common_download_opts opts; + opts.offline = true; + for (const auto & s : sidecars) { + opts.*s.opt = true; + } + + std::unordered_set tags; + std::unordered_set used; + + for (const auto & f : hf_cache::get_cached_files(repo_id)) { + auto split = get_gguf_split_info(f.path); + if (!is_listed_model(split) || split.tag == tag || !tags.insert(split.tag).second) { + continue; + } + + common_params_model model; + model.hf_repo = repo_id + ":" + split.tag; + + auto plan = common_download_get_hf_plan(model, opts); + + for (const auto & m : plan.model_files) { + used.insert(m.local_path); + } + for (const auto & s : sidecars) { + const auto & sf = plan.*s.file; + if (!sf.local_path.empty()) { + used.insert(sf.local_path); + } + } + } + return used; +} + +static bool remove_snapshot_entries(const std::string & repo_id, const std::vector & to_remove); + bool common_download_remove(const std::string & hf_repo_with_tag) { namespace fs = std::filesystem; @@ -989,11 +1075,13 @@ bool common_download_remove(const std::string & hf_repo_with_tag) { return false; } - // collect snapshot entries whose tag matches + // collect snapshot entries whose tag matches, but a file another tag loads stays + const auto used = files_in_use(repo_id, tag_upper); + std::vector to_remove; for (const auto & f : files) { auto split = get_gguf_split_info(f.path); - if (split.tag == tag_upper) { + if (split.tag == tag_upper && used.find(f.local_path) == used.end()) { to_remove.emplace_back(f.local_path); } } @@ -1002,6 +1090,13 @@ bool common_download_remove(const std::string & hf_repo_with_tag) { return false; } + return remove_snapshot_entries(repo_id, to_remove); +} + +// remove snapshot entries of `repo_id`, then the blobs no remaining entry references +static bool remove_snapshot_entries(const std::string & repo_id, const std::vector & to_remove) { + namespace fs = std::filesystem; + // resolve blob paths from symlinks before deleting snapshot entries std::vector blobs_to_check; for (const auto & p : to_remove) { @@ -1053,3 +1148,26 @@ bool common_download_remove(const std::string & hf_repo_with_tag) { return true; } + +bool common_download_remove_sidecars(const std::string & hf_repo_with_tag, const common_download_opts & opts) { + common_params_model model; + model.hf_repo = hf_repo_with_tag; + + common_download_opts cached = opts; + cached.offline = true; + + auto plan = common_download_get_hf_plan(model, cached); + + std::vector to_remove; + for (const auto & s : sidecars) { + const auto & f = plan.*s.file; + if (!f.local_path.empty()) { + to_remove.emplace_back(std::filesystem::u8path(f.local_path)); + } + } + if (to_remove.empty()) { + return false; + } + + return remove_snapshot_entries(common_download_split_repo_tag(hf_repo_with_tag).first, to_remove); +} diff --git a/common/download.h b/common/download.h index 8c30cfc3eadb..4dfd0c9831ed 100644 --- a/common/download.h +++ b/common/download.h @@ -104,10 +104,17 @@ std::string common_docker_resolve_model(const std::string & docker); // Remove a cached model from disk // input format: "user/model" or "user/model:tag" // - if tag is omitted, removes the entire repo cache directory -// - if tag is present, removes only files matching that tag (and orphaned blobs) +// - if tag is present, removes only files matching that tag (and orphaned blobs), +// except the files another cached tag of the repo still loads // returns true if anything was removed bool common_download_remove(const std::string & hf_repo_with_tag); +// remove the cached sidecar files `opts` selects for "user/model:tag", as the cached +// plan resolves them, and leave the model files in place; a sidecar file another tag +// shares goes for every tag that resolves it +// returns true if anything was removed +bool common_download_remove_sidecars(const std::string & hf_repo_with_tag, const common_download_opts & opts); + struct common_download_hf_plan { hf_cache::hf_file primary; hf_cache::hf_files model_files; @@ -119,3 +126,33 @@ struct common_download_hf_plan { hf_cache::hf_file preset; // if set, only this file is downloaded }; common_download_hf_plan common_download_get_hf_plan(const common_params_model & model, const common_download_opts & opts); + +// a sidecar kind a download can pull next to its model: mmproj, and the draft heads +// of --spec-type draft- +struct common_download_sidecar { + const char * name; // the kind, as the API names it + const char * keyword; // the part of a file name that marks a file of this kind + bool draft; // a draft head, resolved at the tag even without a model + + bool common_download_opts::* opt; // the option that requests it + hf_cache::hf_file common_download_hf_plan::* file; // the plan file it resolves to +}; + +// every sidecar kind, the single list the resolution, the listing and the removal follow +const std::vector & common_download_sidecars(); + +// select the sidecar `kind` in `opts`; returns false for an unknown kind +bool common_download_opts_add_sidecar(common_download_opts & opts, const std::string & kind); + +// the sidecar kinds `plan` resolves a file for +std::vector common_download_plan_sidecars(const common_download_hf_plan & plan); + +// a file `plan` still has to download, at the path its partial takes while it is +// incomplete; kind is the sidecar it belongs to, empty for the model files +struct common_download_partial { + std::string kind; + std::string path; +}; + +// the partials the download of `plan` writes, for the files not in the cache yet +std::vector common_download_plan_partials(const common_download_hf_plan & plan); diff --git a/tests/test-model-resolution.cpp b/tests/test-model-resolution.cpp index 5c793e24a6ca..eb7b30936e29 100644 --- a/tests/test-model-resolution.cpp +++ b/tests/test-model-resolution.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -339,6 +340,124 @@ static void test_plan_resolution() { g_repos.clear(); } +// put a file in the cache the way a finished download leaves it, refs/main included +static void cache_put(const std::string & repo_id, const std::string & path) { + namespace fs = std::filesystem; + + const fs::path local = cached(repo_id, path); + fs::create_directories(local.parent_path()); + { std::ofstream(local) << "gguf"; } + + const fs::path ref = local.parent_path().parent_path().parent_path() / "refs" / "main"; + fs::create_directories(ref.parent_path()); + if (!fs::exists(ref)) { + std::ofstream(ref) << COMMIT << "\n"; + } +} + +// +// partials of a download: the files of its plan not in the cache yet, next to +// the blobs, named after the oids the listing serves +// + +static void test_plan_partials() { + namespace fs = std::filesystem; + + printf("test-model-resolution: plan partials\n"); + + g_context = "plan partials"; + g_repos.clear(); + g_repos["test/partial"] = { "model-Q4_0.gguf", "mmproj-model-F16.gguf", "mtp-model-Q4_0.gguf" }; + + // the mmproj is complete, the model and its mtp are still to download + cache_put("test/partial", "mmproj-model-F16.gguf"); + + common_download_opts opts; + opts.download_mmproj = true; + opts.download_mtp = true; + + const auto parts = common_download_plan_partials(common_download_get_hf_plan(model_ref("test/partial:Q4_0"), opts)); + + const fs::path blobs = cache_dir / "models--test--partial" / "blobs"; + REQUIRE(parts.size() == 2); + REQUIRE(parts[0].kind.empty()); + REQUIRE(fs::path(parts[0].path).lexically_normal() == (blobs / "0000000000000000000000000000000000000001.downloadInProgress").lexically_normal()); + REQUIRE_EQ(parts[1].kind, "mtp"); + REQUIRE(fs::path(parts[1].path).lexically_normal() == (blobs / "0000000000000000000000000000000000000003.downloadInProgress").lexically_normal()); + + fs::remove_all(cache_dir / "models--test--partial"); + g_repos.clear(); +} + +// +// removal of a sidecar alone: its snapshot entry goes, the model keeps its files +// + +static void test_sidecar_removal() { + namespace fs = std::filesystem; + + printf("test-model-resolution: sidecar removal\n"); + + g_context = "sidecar removal"; + g_repos.clear(); + + cache_put("test/sidecar", "model-Q4_0.gguf"); + cache_put("test/sidecar", "dflash-model-Q4_0.gguf"); + cache_put("test/sidecar", "mtp-model-Q4_0.gguf"); + + common_download_opts opts; + opts.download_dflash = true; + + // the selected sidecar goes, the other one and the model stay + REQUIRE(common_download_remove_sidecars("test/sidecar:Q4_0", opts)); + REQUIRE(!fs::exists(cached("test/sidecar", "dflash-model-Q4_0.gguf"))); + REQUIRE(fs::exists(cached("test/sidecar", "mtp-model-Q4_0.gguf"))); + REQUIRE(fs::exists(cached("test/sidecar", "model-Q4_0.gguf"))); + REQUIRE(!common_download_remove_sidecars("test/sidecar:Q4_0", opts)); + + fs::remove_all(cache_dir / "models--test--sidecar"); +} + +// +// removal of a tag keeps the files another cached tag still loads: a sidecar +// named after the removed tag that another quant resolves stays on disk +// + +static void test_shared_removal() { + namespace fs = std::filesystem; + + printf("test-model-resolution: shared removal\n"); + + g_context = "shared removal"; + g_repos.clear(); + + for (const char * f : { "model-Q8_0.gguf", "model-BF16.gguf", "mmproj-model-BF16.gguf", "mtp-model-BF16.gguf" }) { + cache_put("test/shared", f); + } + + // the BF16 model goes, the mmproj and the mtp Q8_0 resolves stay + REQUIRE(common_download_remove("test/shared:BF16")); + REQUIRE(!fs::exists(cached("test/shared", "model-BF16.gguf"))); + REQUIRE(fs::exists(cached("test/shared", "mmproj-model-BF16.gguf"))); + REQUIRE(fs::exists(cached("test/shared", "mtp-model-BF16.gguf"))); + REQUIRE(fs::exists(cached("test/shared", "model-Q8_0.gguf"))); + + fs::remove_all(cache_dir / "models--test--shared"); + + // with no other tag, the files of the removed tag all go, sidecars included + for (const char * f : { "model-BF16.gguf", "mmproj-model-BF16.gguf", "mtp-model-BF16.gguf" }) { + cache_put("test/alone", f); + } + + REQUIRE(common_download_remove("test/alone:BF16")); + REQUIRE(!fs::exists(cached("test/alone", "model-BF16.gguf"))); + REQUIRE(!fs::exists(cached("test/alone", "mmproj-model-BF16.gguf"))); + REQUIRE(!fs::exists(cached("test/alone", "mtp-model-BF16.gguf"))); + REQUIRE(!common_download_remove("test/alone:BF16")); + + fs::remove_all(cache_dir / "models--test--alone"); +} + // // end-to-end assembly: real CLI parsing, real handler init resolving over the // loopback, downloads skipped by flipping offline before apply @@ -509,6 +628,9 @@ int main(void) { common_set_env("MODEL_ENDPOINT", "http://127.0.0.1:" + std::to_string(port) + "/"); test_plan_resolution(); + test_plan_partials(); + test_sidecar_removal(); + test_shared_removal(); test_task_assembly(); server.stop(); diff --git a/tools/server/README.md b/tools/server/README.md index c42336c7f217..ec82db0076ed 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -2106,6 +2106,32 @@ Note: for "downloading" state, there can be multiple files be downloading in par } ``` +A download that never finished (paused or interrupted) is listed under a separate +"incomplete" status, without an instance: posting the model to `/models` again resumes it +from the files already on disk, and `DELETE /models` drops its partial files. + +```json +"status": { + "value": "incomplete", + "args": [] +} +``` + +Cached models and incomplete downloads carry the sidecar kinds they have on disk +(`mmproj`, `mtp`, `dflash`, `eagle3`, `dspark`), resolved the way `-hf` loads them: + +```json +"sidecars": ["mmproj", "mtp"] +``` + +A cached model whose download of more sidecars never finished also lists the kinds not on +disk yet: posting the model with them resumes them, and `DELETE /models` with `&sidecar={kind}` +drops their partial files. + +```json +"incomplete_sidecars": ["mtp"] +``` + ### POST `/models/load`: Load a model Load a model @@ -2225,7 +2251,7 @@ Example events: Trigger a new download (non-blocking), the progress can be tracked via SSE endpoint `/models/sse` -To cancel model downloading, send an event to `/models/unload` +To pause a download, send an event to `/models/unload`: it keeps its partial files and is listed as "incomplete" until it is posted again. To cancel it and drop its files, use `DELETE /models`. Download procedure: - Send POST request to `/models` @@ -2238,9 +2264,14 @@ Payload: ```json { "model": "ggml-org/gemma-3-4b-it-GGUF:Q4_K_M", + "sidecars": ["mmproj", "mtp"] } ``` +`sidecars` lists the sidecar kinds pulled with the model, picked by the server the way `-hf` picks them: `mmproj`, and the draft heads `mtp`, `dflash`, `eagle3` and `dspark`. It defaults to `["mmproj"]`, as `-hf` does. + +Posting a model that is already in the cache and not running downloads the sidecars it lacks and keeps the files already on disk. + Response (download is started in the background): ```json @@ -2267,6 +2298,10 @@ IMPORTANT: only model stored in cache can be deleted. You cannot delete models i Model name must be passed via query param: `?model={name}` +To remove one sidecar of a cached model and keep the model, add `&sidecar={kind}` (`mmproj`, `mtp`, `dflash`, `eagle3` or `dspark`); the model must not be running. + +Deleting a model keeps the files another cached quant of the repo still loads, such as an mmproj named after the deleted quant, and the partial files another download still writes. A sidecar deleted on its own goes for every quant that shares it. + If delete success, it will send an SSE event of type `model_remove` Response: diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 60353240ac91..d77837c866f8 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -618,6 +619,7 @@ server_models::server_models( LOG_WRN("failed to get server executable path: %s\n", e.what()); LOG_WRN("using original argv[0] as fallback: %s\n", argv[0]); } + unfinished_downloads.load(); load_models(); debug_fake_timing = !common_get_env("LLAMA_SERVER_DEBUG_FAKE_TIMING").empty(); } @@ -686,6 +688,233 @@ void server_models::add_model(server_model_meta && meta) { }; } +static bool has_sidecar(const std::vector & sidecars, const std::string & kind) { + return std::find(sidecars.begin(), sidecars.end(), kind) != sidecars.end(); +} + +// reject a sidecar kind no download can pull +static void check_sidecar(const std::string & kind) { + std::vector kinds; + for (const auto & s : common_download_sidecars()) { + kinds.emplace_back(s.name); + } + if (!has_sidecar(kinds, kind)) { + throw std::invalid_argument("unknown sidecar '" + kind + "', expected one of " + string_join(kinds, ", ")); + } +} + +// the sidecars a download asks for; by default the ones -hf pulls without a draft type +static std::vector parse_sidecars(const json & body) { + std::vector defaults; + for (const auto & s : common_download_sidecars()) { + if (!s.draft) { + defaults.emplace_back(s.name); + } + } + std::vector sidecars = json_value(body, "sidecars", defaults); + for (const auto & kind : sidecars) { + check_sidecar(kind); + } + return sidecars; +} + +// the download options of a model with its sidecars, the plan a removal purges partials from +static common_download_opts sidecar_download_opts(const common_params & params, const std::vector & sidecars) { + common_download_opts opts; + opts.bearer_token = params.hf_token; + for (const auto & kind : sidecars) { + common_download_opts_add_sidecar(opts, kind); + } + return opts; +} + +// the HF repo a model resolves from: a preset names its own, a cached model is +// listed under it, and a local file or a models dir entry has none +static std::string model_hf_repo(const server_model_meta & meta) { + std::string hf_repo; + if (meta.preset.get_option("LLAMA_ARG_HF_REPO", hf_repo) && !hf_repo.empty()) { + return hf_repo; + } + return meta.source == SERVER_MODEL_SOURCE_CACHE ? meta.name : ""; +} + +// the sidecar kinds of a HF repo present on disk, as the offline plan resolves them +static json cached_sidecars(const std::string & hf_repo) { + common_params_model model; + model.hf_repo = hf_repo; + + common_download_opts opts; + opts.offline = true; + for (const auto & s : common_download_sidecars()) { + opts.*s.opt = true; + } + + return common_download_plan_sidecars(common_download_get_hf_plan(model, opts)); +} + +// replace any inherited value of `key` in a child environment +static void set_child_env(std::vector & env, const std::string & key, const std::string & value) { + env.erase(std::remove_if(env.begin(), env.end(), [&](const std::string & e) { + return e.rfind(key + "=", 0) == 0; + }), env.end()); + env.push_back(key + "=" + value); +} + +// where the unfinished downloads are remembered, inside the model cache +static std::string unfinished_downloads_path() { + return hf_cache::get_cache_path() + "/unfinished-downloads.json"; +} + +void server_unfinished_downloads::load() { + std::ifstream in(unfinished_downloads_path()); + if (!in) { + return; + } + std::string raw((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + std::lock_guard lk(mutex); + try { + json data = json::parse(raw); + for (const auto & d : data.value("downloads", json::array())) { + server_unfinished_download download; + download.model = json_value(d, "model", std::string()); + download.sidecars = json_value(d, "sidecars", std::vector()); + for (const auto & p : d.value("partials", json::array())) { + download.partials.push_back({ json_value(p, "kind", std::string()), json_value(p, "path", std::string()) }); + } + if (!download.model.empty()) { + downloads.push_back(std::move(download)); + } + } + } catch (const std::exception & e) { + LOG_WRN("failed to read unfinished downloads: %s\n", e.what()); + } +} + +void server_unfinished_downloads::save() { + json data; + data["downloads"] = json::array(); + for (const auto & d : downloads) { + json partials = json::array(); + for (const auto & p : d.partials) { + partials.push_back({ {"kind", p.kind}, {"path", p.path} }); + } + data["downloads"].push_back({ {"model", d.model}, {"sidecars", d.sidecars}, {"partials", partials} }); + } + try { + fs_write_atomic(std::filesystem::u8path(unfinished_downloads_path()), data.dump()); + } catch (const std::exception & e) { + LOG_WRN("failed to record unfinished downloads: %s\n", e.what()); + } +} + +void server_unfinished_downloads::delete_partials(const std::vector & parts) { + for (const auto & p : parts) { + const bool shared = std::any_of(downloads.begin(), downloads.end(), [&](const server_unfinished_download & d) { + return std::any_of(d.partials.begin(), d.partials.end(), [&](const common_download_partial & q) { + return q.path == p.path; + }); + }); + if (shared) { + continue; + } + std::error_code ec; + std::filesystem::remove(std::filesystem::u8path(p.path), ec); + } +} + +void server_unfinished_downloads::add(const server_unfinished_download & download) { + std::lock_guard lk(mutex); + std::vector stale; + auto it = std::find_if(downloads.begin(), downloads.end(), [&](const auto & d) { return d.model == download.model; }); + if (it != downloads.end()) { + stale = std::move(it->partials); + *it = download; + } else { + downloads.push_back(download); + } + save(); + delete_partials(stale); +} + +void server_unfinished_downloads::forget(const std::string & model) { + std::lock_guard lk(mutex); + auto it = std::find_if(downloads.begin(), downloads.end(), [&](const auto & d) { return d.model == model; }); + if (it != downloads.end()) { + downloads.erase(it); + save(); + } +} + +bool server_unfinished_downloads::remove(const std::string & model) { + std::lock_guard lk(mutex); + auto it = std::find_if(downloads.begin(), downloads.end(), [&](const auto & d) { return d.model == model; }); + if (it == downloads.end()) { + return false; + } + auto partials = std::move(it->partials); + downloads.erase(it); + save(); + delete_partials(partials); + return true; +} + +bool server_unfinished_downloads::remove_sidecar(const std::string & model, const std::string & kind) { + std::lock_guard lk(mutex); + auto it = std::find_if(downloads.begin(), downloads.end(), [&](const auto & d) { return d.model == model; }); + if (it == downloads.end() || !has_sidecar(it->sidecars, kind)) { + return false; + } + it->sidecars.erase(std::remove(it->sidecars.begin(), it->sidecars.end(), kind), it->sidecars.end()); + + std::vector dropped; + auto & partials = it->partials; + auto pos = std::stable_partition(partials.begin(), partials.end(), [&](const common_download_partial & p) { + return p.kind != kind; + }); + dropped.assign(pos, partials.end()); + partials.erase(pos, partials.end()); + + save(); + delete_partials(dropped); + return true; +} + +std::vector server_unfinished_downloads::get_all() { + std::lock_guard lk(mutex); + return downloads; +} + +std::vector server_models::unfinished_downloads_get() { + return unfinished_downloads.get_all(); +} + +void server_models::unfinished_downloads_remove(const std::string & name) { + // under the lock, so a download of the same tag cannot start while its files go + std::lock_guard lk(mutex); + if (mapping.find(name) != mapping.end()) { + throw std::invalid_argument("model '" + name + "' is downloading"); + } + // a name with no record is no model: its files belong to the cached tags of the repo + if (!unfinished_downloads.remove(name)) { + throw std::runtime_error("model name=" + name + " is not found"); + } + bool ok = common_download_remove(name); + SRV_INF("removing unfinished download name=%s (%s)\n", name.c_str(), ok ? "succeeded" : "nothing cached"); +} + +bool server_models::remove_sidecar(const std::string & name, const std::string & kind) { + std::lock_guard lk(mutex); + auto it = mapping.find(name); + if (it != mapping.end() && (it->second.meta.is_running() || it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING)) { + throw std::invalid_argument("model '" + name + "' is in use, unload it first"); + } + // a sidecar that only ever landed as a partial is still something removed + bool ok = unfinished_downloads.remove_sidecar(name, kind); + ok = common_download_remove_sidecars(name, sidecar_download_opts(base_params, { kind })) || ok; + SRV_INF("removing sidecar %s of model name=%s (%s)\n", kind.c_str(), name.c_str(), ok ? "succeeded" : "nothing cached"); + return ok; +} + void server_models::notify_sse(const std::string & event, const std::string & model_id, const json & data) { std::unique_ptr result = std::make_unique(); result->data = { @@ -1135,7 +1364,8 @@ void server_models::load(const std::string & name) { } void server_models::load(const std::string & name, const load_options & opts) { - if (debug_fake_timing) { + // the fake load time of a model; a download worker loads none + if (debug_fake_timing && opts.mode == SERVER_CHILD_MODE_NORMAL) { // do not hold the mutex here, other requests must keep making progress std::this_thread::sleep_for(std::chrono::seconds(2)); } @@ -1152,6 +1382,43 @@ void server_models::load(const std::string & name, const load_options & opts) { // against the freshest preset and a consistent mapping state cv.wait(lk, [this]() { return !is_reloading; }); + // a download over an existing entry is decided under the lock, so a load of the + // same model cannot start in between: a paused download or an idle cached model + // is downloaded again with its files kept, a download on its way and a running + // model are refused, and a preset or local model is not a download + if (opts.mode == SERVER_CHILD_MODE_DOWNLOAD) { + auto it = mapping.find(name); + if (it != mapping.end()) { + const auto & cur = it->second.meta; + if (cur.status == SERVER_MODEL_STATUS_DOWNLOADING) { + throw std::invalid_argument("model '" + name + "' is already downloading"); + } + if (cur.is_running()) { + throw std::invalid_argument("model '" + name + "' is running, unload it first"); + } + if (cur.source != SERVER_MODEL_SOURCE_CACHE) { + throw std::invalid_argument("model '" + name + "' already exists"); + } + SRV_INF("resuming download for model '%s'\n", name.c_str()); + } + + // two downloads writing the same partial would interleave their bytes + for (const auto & d : unfinished_downloads.get_all()) { + auto other = mapping.find(d.model); + if (d.model == name || other == mapping.end() || other->second.meta.status != SERVER_MODEL_STATUS_DOWNLOADING) { + continue; + } + for (const auto & p : d.partials) { + const bool shared = std::any_of(opts.partials.begin(), opts.partials.end(), [&](const common_download_partial & q) { + return q.path == p.path; + }); + if (shared) { + throw std::invalid_argument("model '" + name + "' shares files with the download of '" + d.model + "' on its way, retry once it is done"); + } + } + } + } + auto meta = opts.custom_meta.has_value() ? *opts.custom_meta : mapping[name].meta; if (meta.status != SERVER_MODEL_STATUS_UNLOADED) { SRV_INF("model %s is not ready\n", name.c_str()); @@ -1201,6 +1468,23 @@ void server_models::load(const std::string & name, const load_options & opts) { inst.meta.status = SERVER_MODEL_STATUS_DOWNLOADING; child_env.push_back("LLAMA_SERVER_CHILD_MODE=download"); child_env.push_back("LLAMA_ARG_HF_REPO=" + name); + + // the sidecars as -hf would pull them: the draft types, and the mmproj switch + // for the one that is not a draft + bool mmproj = false; + std::string spec_types; + for (const auto & s : common_download_sidecars()) { + if (!has_sidecar(opts.sidecars, s.name)) { + continue; + } + if (s.draft) { + spec_types += (spec_types.empty() ? "" : ",") + std::string("draft-") + s.name; + } else { + mmproj = true; + } + } + set_child_env(child_env, "LLAMA_ARG_MMPROJ_AUTO", mmproj ? "1" : "0"); + set_child_env(child_env, "LLAMA_ARG_SPEC_TYPE", spec_types.empty() ? "none" : spec_types); } SRV_INF("%s", "spawning server instance with args:\n"); @@ -1216,6 +1500,11 @@ void server_models::load(const std::string & name, const load_options & opts) { } } + // remember the download: the partial files carry a content hash, not the name + if (opts.mode == SERVER_CHILD_MODE_DOWNLOAD) { + unfinished_downloads.add({ name, opts.sidecars, opts.partials }); + } + // old process should have exited already, but just in case, we clean it up here { auto it = mapping.find(name); @@ -1390,6 +1679,12 @@ void server_models::update_download_progress(const std::string & name, const com // mark the instance to be erased on next load_models() call it->second.meta.status = SERVER_MODEL_STATUS_DOWNLOADED; need_reload = true; + + // a download that failed keeps its tag: its partial files are still + // there and re-posting the tag resumes them + if (ok) { + unfinished_downloads.forget(name); + } } else { json & info = it->second.meta.loaded_info; if (!info.contains("progress")) { @@ -1445,24 +1740,14 @@ bool server_models::remove(const std::string & name) { || meta.status == SERVER_MODEL_STATUS_DOWNLOADED; }); - // re-find after wait - load_models() may have erased the entry during the wait - it = mapping.find(name); - if (it == mapping.end()) { - // load_models() already erased the entry; we just need to clean up the cached files on disk - lk.unlock(); - bool ok = common_download_remove(name); - SRV_INF("removing model name=%s from cache (%s)\n", name.c_str(), ok ? "succeeded" : "partial"); - notify_sse("model_remove", name, {}); - return true; - } + // load_models() may have erased the entry during the wait; the files go under the + // lock, so a download of the same tag cannot start meanwhile + mapping.erase(name); + unfinished_downloads.remove(name); - // remove from disk (best-effort: cancelled downloads may have no cached files) + // best-effort: a cancelled download may have no cached files bool ok = common_download_remove(name); - mapping.erase(name); - if (!ok) { - SRV_WRN("removing model name=%s from disk returned false (no cached files?)\n", name.c_str()); - } - SRV_INF("removing model name=%s from cache (%s)\n", name.c_str(), ok ? "succeeded" : "partial"); + SRV_INF("removing model name=%s from cache (%s)\n", name.c_str(), ok ? "succeeded" : "nothing cached"); notify_sse("model_remove", name, {}); return true; } @@ -2091,6 +2376,9 @@ void server_models_routes::init_routes() { json models_json = json::array(); auto all_models = models.get_all_meta(); std::time_t t = std::time(0); + // a download on its way has no files to resolve yet, only the kinds it pulls + const auto unfinished = models.unfinished_downloads_get(); + for (const auto & meta : all_models) { if (meta.hidden) { continue; // cache model deduplicated by a preset @@ -2132,6 +2420,35 @@ void server_models_routes::init_routes() { model_info["context_length"] = meta.n_ctx_train; } + // the sidecars a model has on disk, resolved as -hf would load them, for a + // model that resolves from a HF repo. A download on its way reports the + // kinds it pulls, as an unfinished one does + const auto it_unfinished = std::find_if(unfinished.begin(), unfinished.end(), + [&](const server_unfinished_download & d) { return d.model == meta.name; }); + const std::string hf_repo = model_hf_repo(meta); + + if (it_unfinished != unfinished.end() && meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { + model_info["sidecars"] = it_unfinished->sidecars; + } else if (!hf_repo.empty()) { + const json cached = cached_sidecars(hf_repo); + model_info["sidecars"] = cached; + + // the sidecars of an unfinished download of this model that are not on + // disk yet: a POST with the kind resumes one, a DELETE with it drops it + if (it_unfinished != unfinished.end()) { + std::vector incomplete; + for (const auto & p : it_unfinished->partials) { + if (!p.kind.empty() && !has_sidecar(incomplete, p.kind) && + std::find(cached.begin(), cached.end(), p.kind) == cached.end()) { + incomplete.push_back(p.kind); + } + } + if (!incomplete.empty()) { + model_info["incomplete_sidecars"] = incomplete; + } + } + } + // merge with loaded_info from the child process if available if (meta.is_running()) { for (auto it = meta.loaded_info.begin(); it != meta.loaded_info.end(); ++it) { @@ -2142,6 +2459,33 @@ void server_models_routes::init_routes() { } models_json.push_back(model_info); } + + const json architecture_default = json { + {"input_modalities", json::array({"text"})}, + {"output_modalities", json::array({"text"})}, + }; + + // a download that never finished has no instance: report it as incomplete so + // the UI can offer to resume it, without starting anything + for (const auto & d : unfinished) { + if (models.has_model(d.model)) { + continue; // complete, or downloading right now + } + models_json.push_back(json { + {"id", d.model}, + {"aliases", json::array()}, + {"tags", json::array()}, + {"object", "model"}, + {"owned_by", "llamacpp"}, + {"created", t}, + {"status", json { {"value", server_model_status_to_string(SERVER_MODEL_STATUS_INCOMPLETE)}, {"args", json::array()} }}, + {"architecture", architecture_default}, + {"source", server_model_source_to_string(SERVER_MODEL_SOURCE_CACHE)}, + {"can_remove", true}, + {"sidecars", d.sidecars}, + }); + } + res_ok(res, { {"data", models_json}, {"object", "list"}, @@ -2193,32 +2537,27 @@ void server_models_routes::init_routes() { if (name.empty()) { throw std::invalid_argument("model must be a non-empty string"); } + const auto sidecars = parse_sidecars(body); - common_params p; - p.model.hf_repo = name; - p.hf_token = params.hf_token; + common_params_model model; + model.hf_repo = name; - // validate by fetching metadata - bool ok = false; + // validate by fetching metadata, through the plan the download follows + common_download_hf_plan plan; try { - common_models_handler_init(p, LLAMA_EXAMPLE_SERVER); - ok = true; + plan = common_download_get_hf_plan(model, sidecar_download_opts(params, sidecars)); } catch (...) { SRV_ERR("unknown error while validating model '%s'\n", name.c_str()); // other exceptions will be handled by the outer ex_wrapper() throw; } - if (!ok) { - throw std::invalid_argument("model validation failed, unable to download"); - } - - // reject if model already exists - if (models.has_model(name)) { - throw std::invalid_argument("model '" + name + "' already exists"); + // a tag that resolves no file has nothing to download + if (plan.preset.path.empty() && plan.primary.path.empty() && common_download_plan_sidecars(plan).empty()) { + throw std::invalid_argument("model '" + name + "' resolves no file to download"); } - // then, proceed with the actual download + // then, proceed with the actual download; load() decides over an existing entry SRV_INF("starting download for model '%s'\n", name.c_str()); { server_models::load_options load_opts; @@ -2226,6 +2565,8 @@ void server_models_routes::init_routes() { load_opts.custom_meta = server_model_meta{}; load_opts.custom_meta->source = SERVER_MODEL_SOURCE_CACHE; load_opts.custom_meta->name = name; + load_opts.sidecars = sidecars; + load_opts.partials = common_download_plan_partials(plan); models.load(name, load_opts); } @@ -2241,6 +2582,22 @@ void server_models_routes::init_routes() { throw std::invalid_argument("model must be a non-empty string"); } + // a sidecar alone goes, the model keeps its files + std::string sidecar = req.get_param("sidecar"); + if (!sidecar.empty()) { + check_sidecar(sidecar); + res_ok(res, {{"success", models.remove_sidecar(name, sidecar)}}); + return res; + } + + // a download listed from an earlier run has no instance, only its tag and + // its partial files + if (!models.has_model(name)) { + models.unfinished_downloads_remove(name); + res_ok(res, {{"success", true}}); + return res; + } + models.remove(name); // throws on error res_ok(res, {{"success", true}}); diff --git a/tools/server/server-models.h b/tools/server/server-models.h index f415b7c5fd61..b04fc7384d9f 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -16,6 +16,7 @@ #include #include #include +#include /** * state diagram: @@ -31,6 +32,7 @@ enum server_model_status { // TODO: also add downloading state when the logic is added SERVER_MODEL_STATUS_DOWNLOADING, + SERVER_MODEL_STATUS_INCOMPLETE, // a download that never finished, listed without an instance SERVER_MODEL_STATUS_DOWNLOADED, SERVER_MODEL_STATUS_UNLOADED, SERVER_MODEL_STATUS_LOADING, @@ -52,6 +54,7 @@ enum server_child_mode { static std::string server_model_status_to_string(server_model_status status) { switch (status) { case SERVER_MODEL_STATUS_DOWNLOADING: return "downloading"; + case SERVER_MODEL_STATUS_INCOMPLETE: return "incomplete"; case SERVER_MODEL_STATUS_DOWNLOADED: return "downloaded"; case SERVER_MODEL_STATUS_UNLOADED: return "unloaded"; case SERVER_MODEL_STATUS_LOADING: return "loading"; @@ -108,6 +111,51 @@ struct server_model_meta { void update_caps(const common_params & base); }; +// a download that never finished: the tag it was asked under, the sidecar kinds it +// pulls and the partial files its plan writes +struct server_unfinished_download { + std::string model; + std::vector sidecars; + std::vector partials; +}; + +// the downloads that never finished, recorded next to the model cache. The cache names +// partial files after their content hash, so this record is the only trace of what is +// still incomplete: it lets a restarted router list, resume or remove them +// a partial another recorded download also writes is never deleted +// these functions are thread-safe +struct server_unfinished_downloads { + // read the record an earlier run left + void load(); + + // record a download that starts, replacing an earlier record of the same model, + // and delete the partials of that record the new plan no longer writes + void add(const server_unfinished_download & download); + + // forget a download whose files are complete + void forget(const std::string & model); + + // forget a download and delete its partials; returns false when it is not recorded + bool remove(const std::string & model); + + // drop the sidecar `kind` from a download and delete its partials; returns false + // when the download is not recorded with that kind + bool remove_sidecar(const std::string & model, const std::string & kind); + + // every recorded download + std::vector get_all(); + +private: + std::mutex mutex; + std::vector downloads; + + // write the record back; callers hold the mutex + void save(); + + // delete the partials of `parts` no recorded download writes; callers hold the mutex + void delete_partials(const std::vector & parts); +}; + struct server_models_routes; struct server_lru_sched; // defined in server-models.cpp struct server_monitor; // defined in server-models.cpp @@ -134,6 +182,9 @@ struct server_models { // models asked to stop, still counted as running until the monitor records their exit std::set stopping_models; + // downloads that never finished, with the sidecar kinds they pull + server_unfinished_downloads unfinished_downloads; + // set to true while load_models() is executing a reload; load() will wait until clear bool is_reloading = false; @@ -252,6 +303,16 @@ struct server_models { // check if a model instance exists (thread-safe) bool has_model(const std::string & name); + // downloads that never finished, with their sidecar kinds (thread-safe) + std::vector unfinished_downloads_get(); + + // drop a download that will never be resumed, its record and its partial files; + // throws for a name that is not recorded (thread-safe) + void unfinished_downloads_remove(const std::string & name); + + // remove the sidecar `kind` of a model that is not in use, its files and partials (thread-safe) + bool remove_sidecar(const std::string & name, const std::string & kind); + // return a copy of model metadata (thread-safe) std::optional get_meta(const std::string & name); @@ -262,6 +323,10 @@ struct server_models { server_child_mode mode = SERVER_CHILD_MODE_NORMAL; // used for spawning a downloading child process std::optional custom_meta = std::nullopt; + // download mode: the sidecar kinds pulled with the model, as -hf would, and + // the partial files the download plan writes + std::vector sidecars; + std::vector partials; }; // load and unload model instances diff --git a/tools/server/tests/unit/test_router.py b/tools/server/tests/unit/test_router.py index 5355ab7367ed..6df41f8d63e6 100644 --- a/tools/server/tests/unit/test_router.py +++ b/tools/server/tests/unit/test_router.py @@ -1,3 +1,4 @@ +import shutil import threading import pytest from utils import * @@ -546,6 +547,15 @@ def _wait_for_sse_event(collected: list, event_type: str, model: str, timeout: i return False +def _own_cache(name: str) -> str: + """An empty model cache for one test: the workers run in parallel, and a download + test must not see the files or the record another test writes in the shared cache.""" + path = os.path.join(TMP_DIR, name) + shutil.rmtree(path, ignore_errors=True) + os.makedirs(path) + return path + + def test_router_download_model(): """Case 1: download a model at the model limit, verify SSE events and GET /models.""" global server @@ -626,3 +636,230 @@ def test_router_delete_model(): # Model should no longer appear in GET /models ids = _get_model_ids(is_reload=False) assert MODEL_DOWNLOAD_ID not in ids, f"{MODEL_DOWNLOAD_ID} still present after deletion" + + +def test_router_download_sidecars(): + """Case 3: sidecars are a parameter of the download, never models of their own.""" + global server + server.llama_cache = _own_cache("cache_download_sidecars") + server.start() + + server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}") + + # an unknown sidecar kind is rejected before anything starts + res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID, "sidecars": ["lora"]}) + assert res.status_code == 400 + assert MODEL_DOWNLOAD_ID not in _get_model_ids(is_reload=False) + + sse_events: list = [] + stop = threading.Event() + sse_ready = threading.Event() + threading.Thread( + target=_listen_sse, args=(server, sse_events, stop, sse_ready), daemon=True + ).start() + assert sse_ready.wait(10), "SSE client failed to connect" + + res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID, "sidecars": []}) + assert res.status_code == 200 + finished = _wait_for_sse_event( + sse_events, "download_finished", MODEL_DOWNLOAD_ID, MODEL_DOWNLOAD_TIMEOUT + ) + stop.set() + assert finished, "Never received download_finished SSE event" + + # the model lists the sidecars it has on disk, and no sidecar is listed as a model + res = server.make_request("GET", "/models") + assert res.status_code == 200 + entries = {item["id"]: item for item in res.body.get("data", [])} + assert entries[MODEL_DOWNLOAD_ID]["sidecars"] == [] + for kind in ("mmproj", "mtp", "dflash", "eagle3", "dspark"): + assert not any(i.endswith("-" + kind) for i in entries), f"a {kind} sidecar is listed as a model" + + server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}") + + +def test_router_download_conflicts(): + """A download is refused over one on its way, and over a model that is running.""" + global server + server.models_max = 2 + server.llama_cache = _own_cache("cache_download_conflicts") + server.start() + + server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}") + + # a download on its way is not started again; the entry reports the state before + # the child fetches anything, so the probe lands inside the download + res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID}) + assert res.status_code == 200 + _wait_for_model_status(MODEL_DOWNLOAD_ID, {"downloading"}, timeout=30) + + res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID}) + assert res.status_code == 400 + assert "is already downloading" in res.body["error"]["message"] + + # a loaded model holds its files, so it goes only once it is unloaded; the + # reload that follows the download parks the entry as unloaded + _wait_for_model_status(MODEL_DOWNLOAD_ID, {"downloaded", "unloaded"}, timeout=MODEL_DOWNLOAD_TIMEOUT) + _load_model_and_wait(MODEL_DOWNLOAD_ID, timeout=120) + + res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID}) + assert res.status_code == 400 + assert "is running, unload it first" in res.body["error"]["message"] + + +def test_router_delete_unknown_name(): + """A name that is neither a model nor a recorded download is not found, and the + cached tags of its repo keep their files.""" + global server + server.llama_cache = _own_cache("cache_delete_unknown_name") + server.start() + + res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID, "sidecars": []}) + assert res.status_code == 200 + _wait_for_model_status(MODEL_DOWNLOAD_ID, {"downloaded", "unloaded"}, timeout=MODEL_DOWNLOAD_TIMEOUT) + + repo = MODEL_DOWNLOAD_ID.split(":")[0] + res = server.make_request("DELETE", f"/models?model={repo}") + assert res.status_code != 200 + assert "is not found" in res.body["error"]["message"] + assert MODEL_DOWNLOAD_ID in _get_model_ids(is_reload=True) + + server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}") + + +def test_router_incomplete_sidecars(): + """A cached model lists the sidecars its unfinished download has not written yet, until + they are dropped.""" + global server + cache_dir = _own_cache("cache_incomplete_sidecars") + server.llama_cache = cache_dir + server.start() + + res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID, "sidecars": []}) + assert res.status_code == 200 + _wait_for_model_status(MODEL_DOWNLOAD_ID, {"downloaded", "unloaded"}, timeout=MODEL_DOWNLOAD_TIMEOUT) + server.stop() + + # a paused mtp of the cached model, as the record of an earlier run holds it + partial_path = os.path.abspath(os.path.join(cache_dir, "test-mtp.gguf.downloadInProgress")) + with open(partial_path, "w") as f: + f.write("partial") + with open(os.path.join(cache_dir, "unfinished-downloads.json"), "w") as f: + json.dump({"downloads": [{ + "model": MODEL_DOWNLOAD_ID, + "sidecars": ["mtp"], + "partials": [{"kind": "mtp", "path": partial_path}], + }]}, f) + + server.start() + + def entry() -> dict: + res = server.make_request("GET", "/models") + assert res.status_code == 200 + return {item["id"]: item for item in res.body.get("data", [])}[MODEL_DOWNLOAD_ID] + + assert entry()["sidecars"] == [] + assert entry()["incomplete_sidecars"] == ["mtp"] + + res = server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}&sidecar=mtp") + assert res.status_code == 200 + assert res.body.get("success") is True + assert not os.path.exists(partial_path) + assert "incomplete_sidecars" not in entry() + + server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}") + + +def test_router_preset_model_sidecars(): + """A preset model reports the sidecars its files have on disk, like a cached one.""" + global server + + preset_path = os.path.join(TMP_DIR, "test_preset_sidecars.ini") + + with open(preset_path, "w") as f: + f.write("[preset-sidecars]\nhf-repo = ggml-org/tinygemma3-GGUF:Q8_0\n") + + server.models_preset = preset_path + server.start() + + try: + # the mmproj of that repo lands next to the model, as -hf would pull it + res = server.make_request("POST", "/models", data={"model": MODEL_A, "sidecars": ["mmproj"]}) + assert res.status_code == 200 + _wait_for_model_status(MODEL_A, {"downloaded", "unloaded"}, timeout=MODEL_DOWNLOAD_TIMEOUT) + + res = server.make_request("GET", "/models") + assert res.status_code == 200 + entries = {item["id"]: item for item in res.body.get("data", [])} + + assert entries["preset-sidecars"]["source"] == "preset" + assert entries["preset-sidecars"]["sidecars"] == ["mmproj"] + finally: + os.remove(preset_path) + + +def test_router_local_preset_listing(): + """A preset on a local file resolves from no HF repo: it lists without sidecars.""" + global server + + model_path = os.path.join(TMP_DIR, "test_local_preset.gguf") + preset_path = os.path.join(TMP_DIR, "test_local_preset.ini") + + with open(model_path, "w") as f: + f.write("GGUF") + with open(preset_path, "w") as f: + f.write(f"[local]\nmodel = {model_path}\n") + + server.models_preset = preset_path + server.start() + + try: + res = server.make_request("GET", "/models") + assert res.status_code == 200 + entries = {item["id"]: item for item in res.body.get("data", [])} + + assert entries["local"]["source"] == "preset" + assert "sidecars" not in entries["local"] + finally: + os.remove(preset_path) + os.remove(model_path) + + +def test_router_download_unresolved_tag(): + """A tag that resolves no file of the repo is refused before anything starts.""" + global server + server.start() + + model = MODEL_DOWNLOAD_ID.split(":")[0] + ":Q9_Z" + res = server.make_request("POST", "/models", data={"model": model}) + assert res.status_code == 400 + assert "resolves no file to download" in res.body["error"]["message"] + assert model not in _get_model_ids(is_reload=False) + + +def test_router_download_drops_stale_partials(): + """Posting a recorded download again deletes the partials its new plan no longer writes.""" + global server + + cache_dir = _own_cache("cache_download_drops_stale_partials") + stale_path = os.path.abspath(os.path.join(cache_dir, "test-stale.gguf.downloadInProgress")) + record_path = os.path.join(cache_dir, "unfinished-downloads.json") + + with open(stale_path, "w") as f: + f.write("stale") + with open(record_path, "w") as f: + json.dump({"downloads": [{ + "model": MODEL_DOWNLOAD_ID, + "sidecars": ["mtp"], + "partials": [{"kind": "mtp", "path": stale_path}], + }]}, f) + + server.llama_cache = cache_dir + server.start() + + res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID, "sidecars": []}) + assert res.status_code == 200 + assert not os.path.exists(stale_path), "the stale partial survived the new record" + + _wait_for_model_status(MODEL_DOWNLOAD_ID, {"downloaded", "unloaded"}, timeout=MODEL_DOWNLOAD_TIMEOUT) + server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}") diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.py index 1c84f5121145..288452808093 100644 --- a/tools/server/tests/utils.py +++ b/tools/server/tests/utils.py @@ -127,6 +127,7 @@ class ServerProcess: mcp_servers_config: str | None = None mcp_servers_json: str | None = None cors_origins: str | None = None + llama_cache: str | None = None # session variables process: subprocess.Popen | None = None @@ -145,7 +146,9 @@ def start(self, timeout_seconds: int = DEFAULT_HTTP_TIMEOUT) -> None: **os.environ, "LLAMA_SERVER_DEBUG_FAKE_TIMING": "1", } - if "LLAMA_CACHE" not in os.environ: + if self.llama_cache is not None: + env["LLAMA_CACHE"] = self.llama_cache + elif "LLAMA_CACHE" not in os.environ: env["LLAMA_CACHE"] = "tmp" if self.external_server: print(f"[external_server]: Assuming external server running on {self.server_host}:{self.server_port}") diff --git a/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte b/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte index c217a769a6b4..895e9a80a7b1 100644 --- a/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte +++ b/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte @@ -7,6 +7,7 @@ getMdastNodeHash, isAppendMode } from './markdown-utils'; + import { SAFE_HTML_CONFIG } from './safe-html-config'; import { ActionIconCopyToClipboard, CodeBlockActions, @@ -44,6 +45,7 @@ import { detectIncompleteCodeBlock, highlightCode, type IncompleteCodeBlock } from '$lib/utils'; import { sanitizeSvg } from '$lib/utils/sanitize-svg'; import { mountSvgShadow } from '$lib/utils/svg-shadow'; + import DOMPurify from 'dompurify'; import type { Root as HastRoot, RootContent as HastRootContent } from 'hast'; import githubLightCss from 'highlight.js/styles/github.css?inline'; import githubDarkCss from 'highlight.js/styles/github-dark.css?inline'; @@ -57,6 +59,8 @@ content: string; class?: string; disableMath?: boolean; + /** Render raw HTML found in the markdown (sanitized) instead of escaping it. */ + allowHtml?: boolean; } interface MarkdownBlock { @@ -65,7 +69,13 @@ contentHash?: string; } - let { attachments, class: className = '', content, disableMath = false }: Props = $props(); + let { + allowHtml = false, + attachments, + class: className = '', + content, + disableMath = false + }: Props = $props(); let containerRef = $state(); let renderedBlocks = $state([]); @@ -121,6 +131,8 @@ // Per-instance transform cache, avoids re-transforming stable blocks during streaming // Garbage collected when component is destroyed (on conversation change) + const CACHE_KEY_PLAIN = 'plain'; + const CACHE_KEY_ALLOW_HTML = 'allow-html'; const transformCache = new SvelteMap(); let previousContent = ''; @@ -183,7 +195,10 @@ index: number ): Promise<{ html: string; hash: string }> { const hash = getMdastNodeHash(node, index); - const cached = transformCache.get(hash); + // the rendered HTML also depends on allowHtml (sanitized raw vs escaped), + // so the cache is keyed per mode + const cacheKey = `${allowHtml ? CACHE_KEY_ALLOW_HTML : CACHE_KEY_PLAIN}:${hash}`; + const cached = transformCache.get(cacheKey); if (cached) { return { hash, html: cached }; @@ -192,10 +207,13 @@ const singleNodeRoot = { children: [node], type: 'root' }; const transformedRoot = (await processorInstance.run(singleNodeRoot as MdastRoot)) as HastRoot; const html = processorInstance.stringify(transformedRoot); + const safeHtml = allowHtml + ? (DOMPurify.sanitize(html, SAFE_HTML_CONFIG) as unknown as string) + : html; - transformCache.set(hash, html); + transformCache.set(cacheKey, safeHtml); - return { hash, html }; + return { hash, html: safeHtml }; } /** @@ -300,7 +318,7 @@ if (prefixMarkdown.trim()) { const normalizedPrefix = preprocessLaTeX(prefixMarkdown); - const processorInstance = getMarkdownProcessor({ attachments, disableMath }); + const processorInstance = getMarkdownProcessor({ allowHtml, attachments, disableMath }); const ast = processorInstance.parse(normalizedPrefix) as MdastRoot; const mdastChildren = (ast as { children?: unknown[] }).children ?? []; const nextBlocks: MarkdownBlock[] = []; @@ -350,7 +368,7 @@ incompleteCodeBlock = null; const normalized = preprocessLaTeX(markdown); - const processorInstance = getMarkdownProcessor({ attachments, disableMath }); + const processorInstance = getMarkdownProcessor({ allowHtml, attachments, disableMath }); const ast = processorInstance.parse(normalized) as MdastRoot; const mdastChildren = (ast as { children?: unknown[] }).children ?? []; const stableCount = Math.max(mdastChildren.length - 1, 0); @@ -394,6 +412,10 @@ )) as HastRoot; unstableHtml = processorInstance.stringify(transformedRoot); + + if (allowHtml) { + unstableHtml = DOMPurify.sanitize(unstableHtml, SAFE_HTML_CONFIG) as unknown as string; + } } renderedBlocks = nextBlocks; diff --git a/tools/ui/src/lib/components/app/content/MarkdownContent/markdown-processor.ts b/tools/ui/src/lib/components/app/content/MarkdownContent/markdown-processor.ts index 57ded3e99ada..26cae5b5c51b 100644 --- a/tools/ui/src/lib/components/app/content/MarkdownContent/markdown-processor.ts +++ b/tools/ui/src/lib/components/app/content/MarkdownContent/markdown-processor.ts @@ -42,12 +42,15 @@ export interface MarkdownProcessor { export interface MarkdownProcessorOptions { attachments?: DatabaseMessageExtra[]; disableMath?: boolean; + /** Render raw HTML found in the markdown instead of escaping it. */ + allowHtml?: boolean; } const sharedPipelines = new Map(); -const attachmentPipelines = new WeakMap(); +const attachmentPipelines = new WeakMap>(); function buildPipeline({ + allowHtml = false, attachments, disableMath = false }: MarkdownProcessorOptions): MarkdownProcessor { @@ -58,11 +61,14 @@ function buildPipeline({ proc = proc.use(remarkMath); // Parse $inline$ and $$block$$ math } - proc = proc - .use(remarkBreaks) // Convert line breaks to
+ proc = proc.use(remarkBreaks); // Convert line breaks to
+ + if (!allowHtml) { // Treat raw HTML as literal text with preserved indentation - .use(remarkLiteralHtml) - .use(remarkRehype); // Convert Markdown AST to rehype + proc = proc.use(remarkLiteralHtml); + } + + proc = proc.use(remarkRehype, allowHtml ? { allowDangerousHtml: true } : undefined); if (!disableMath) { proc = proc.use(rehypeKatex); // Render math using KaTeX @@ -91,17 +97,26 @@ function buildPipeline({ export function getMarkdownProcessor(options: MarkdownProcessorOptions): MarkdownProcessor { if (options.attachments && options.attachments.length > 0) { - let cached = attachmentPipelines.get(options.attachments); + let byOptions = attachmentPipelines.get(options.attachments); + + if (!byOptions) { + byOptions = new Map(); + attachmentPipelines.set(options.attachments, byOptions); + } + + const key = `${Boolean(options.disableMath)}:${Boolean(options.allowHtml)}`; + + let cached = byOptions.get(key); if (!cached) { cached = buildPipeline(options); - attachmentPipelines.set(options.attachments, cached); + byOptions.set(key, cached); } return cached; } - const key = String(Boolean(options.disableMath)); + const key = `${Boolean(options.disableMath)}:${Boolean(options.allowHtml)}`; let cached = sharedPipelines.get(key); diff --git a/tools/ui/src/lib/components/app/content/MarkdownContent/safe-html-config.ts b/tools/ui/src/lib/components/app/content/MarkdownContent/safe-html-config.ts new file mode 100644 index 000000000000..d8dff07ddaca --- /dev/null +++ b/tools/ui/src/lib/components/app/content/MarkdownContent/safe-html-config.ts @@ -0,0 +1,138 @@ +// Sanitization rules for raw HTML in model cards (MarkdownContent's +// allowHtml mode). Kept as its own module so the adversarial test in +// tests/client can import the exact shipped configuration. +import DOMPurify from 'dompurify'; + +// style is allowed for the KaTeX output, which lays its spans out with inline +// styles; a third party card can use the same attribute to draw a fixed +// overlay over the app, so positioned values are scrubbed +DOMPurify.addHook('afterSanitizeAttributes', (node) => { + if (typeof (node as Element).getAttribute !== 'function') return; + + const style = (node as Element).getAttribute('style'); + + if (!style || !/position\s*:\s*(fixed|absolute)/i.test(style)) return; + + (node as Element).setAttribute( + 'style', + style.replace(/position\s*:\s*(fixed|absolute)\s*;?/gi, '') + ); +}); + +/** + * Explicit DOMPurify allow list: the tags and attributes model cards may use + * plus the KaTeX output (spans with inline styles and MathML). Everything + * else - scripts, event handlers, forms, iframes, style tags - is stripped. + */ +export const SAFE_HTML_CONFIG = { + ALLOWED_ATTR: [ + 'class', + 'style', + 'id', + 'title', + 'role', + 'aria-hidden', + 'aria-label', + 'aria-describedby', + 'aria-expanded', + 'href', + 'target', + 'rel', + 'src', + 'srcset', + 'alt', + 'width', + 'height', + 'align', + 'valign', + 'colspan', + 'rowspan', + 'controls', + 'type', + // MathML + 'xmlns', + 'display', + 'encoding', + 'mathvariant' + ], + ALLOWED_TAGS: [ + // text and structure + 'a', + 'b', + 'i', + 'u', + 's', + 'em', + 'strong', + 'code', + 'pre', + 'kbd', + 'sub', + 'sup', + 'br', + 'hr', + 'p', + 'span', + 'div', + 'blockquote', + 'center', + 'ul', + 'ol', + 'li', + 'dl', + 'dt', + 'dd', + 'h1', + 'h2', + 'h3', + 'h4', + 'h5', + 'h6', + 'table', + 'thead', + 'tbody', + 'tr', + 'th', + 'td', + 'colgroup', + 'col', + 'caption', + // media + 'img', + 'picture', + 'source', + 'figure', + 'figcaption', + 'video', + 'audio', + // MathML, for the KaTeX output + 'math', + 'mrow', + 'mi', + 'mo', + 'mn', + 'ms', + 'mtext', + 'mfrac', + 'mroot', + 'msqrt', + 'msub', + 'msup', + 'msubsup', + 'munder', + 'mover', + 'mmultiscripts', + 'mtable', + 'mtr', + 'mtd', + 'mspace', + 'mglyph', + 'maligngroup', + 'malignmark', + 'mpadded', + 'mphantom', + 'mstyle', + 'semantics', + 'annotation' + ] +}; diff --git a/tools/ui/src/lib/components/app/dialogs/DialogManageModels.svelte b/tools/ui/src/lib/components/app/dialogs/DialogManageModels.svelte index 45b73dc72fc9..fbe711c25cd0 100644 --- a/tools/ui/src/lib/components/app/dialogs/DialogManageModels.svelte +++ b/tools/ui/src/lib/components/app/dialogs/DialogManageModels.svelte @@ -1,7 +1,18 @@ {#each badges as badge (badge.kind + (badge.quant ?? ''))} - - + import { Check } from '@lucide/svelte'; + import { MODEL_BADGE_CLASS, MODEL_VARIANT_BADGE_CLASS } from '$lib/constants'; + import { ModelRowDownloadState } from '$lib/enums'; + + interface Props { + /** + * Left inset of the tree connector, tuned per surface so the elbow hangs + * under the main line's avatar and "Draft model" aligns with its id. + */ + connectorClass?: string; + /** Draft sidecar kind, e.g. dflash. */ + kind: string; + /** Quant of the draft, when its tag carries one. */ + quant?: string | null; + /** Live state of the draft download. */ + state: ModelRowDownloadState; + /** Rounded download percent, when the feed reports progress. */ + percent?: number | null; + } + + let { connectorClass = 'ml-6', kind, percent = null, quant = null, state }: Props = $props(); + + + +
+ + + Draft model + + {kind} + + {#if quant} + {quant} + {/if} + + {#if state === ModelRowDownloadState.DOWNLOADED} + + {:else} + + {#if percent !== null} + {percent}% + {:else if state === ModelRowDownloadState.PAUSED} + Paused + {/if} + + {/if} +
diff --git a/tools/ui/src/lib/components/app/models/ModelId.svelte b/tools/ui/src/lib/components/app/models/ModelId.svelte index 26d72f662ff2..9a3464ab861b 100644 --- a/tools/ui/src/lib/components/app/models/ModelId.svelte +++ b/tools/ui/src/lib/components/app/models/ModelId.svelte @@ -5,6 +5,7 @@ import { TruncatedText } from '$lib/components/app'; import * as Tooltip from '$lib/components/ui/tooltip'; import { MODEL_BADGE_CLASS, MODEL_VARIANT_BADGE_CLASS } from '$lib/constants'; + import type { ModelDraftSidecar } from '$lib/enums'; import { HuggingFaceService } from '$lib/services'; import { ModelsService } from '$lib/services/models.service'; import { settingsStore } from '$lib/stores'; @@ -39,6 +40,8 @@ sizeRange?: { min: number; max: number } | null; /** Draft sidecars available for the model, badged with their own quant. */ draftSidecars?: ModelSidecarBadge[]; + /** Draft sidecar kinds the repo offers, one badge per kind and no quant. */ + draftKinds?: ModelDraftSidecar[]; /** Give the id a row of its own, with its badges and icons together under it. */ stackId?: boolean; /** Allow badges to wrap onto new lines instead of truncating. */ @@ -50,6 +53,7 @@ aliases, class: className = '', contextLength, + draftKinds = [], draftSidecars = [], hideCapabilities = false, hideModalities = false, @@ -93,12 +97,17 @@ let uniqueAliases = $derived([...new Set(aliases ?? [])]); let uniqueTags = $derived([...new Set([...(parsed.tags ?? []), ...(tags ?? [])])]); let uniqueDraftSidecars = $derived(draftSidecars.filter((badge) => !isAuxSidecar(badge.kind))); + // the model id can already name one of the kinds + let uniqueDraftKinds = $derived([ + ...new Set(draftKinds.filter((kind) => kind !== parsed.sidecar)) + ]); let primaryAlias = $derived(uniqueAliases.length === 1 ? uniqueAliases[0] : null); let displayName = $derived(primaryAlias ?? parsed.modelName ?? modelId); let hasBadges = $derived( parsed.sidecar || + uniqueDraftKinds.length > 0 || uniqueDraftSidecars.length > 0 || (parsed.params && !hideParameters) || (parsed.quantization && !resolvedHideQuantization) || @@ -144,6 +153,12 @@ {/if} + {#each uniqueDraftKinds as kind (kind)} + + {kind} + + {/each} + {#if primaryAlias} diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManager.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManager.svelte index 36fd3e22ae2f..9853bafe51a0 100644 --- a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManager.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManager.svelte @@ -32,9 +32,22 @@ let capabilityFilter = $state([]); let selectedId = $state(null); + // tracked downloads get their own section, listed like any other model + let downloads = $derived( + downloadGroups(modelsStore.status.getDownloadEntries(), modelsStore.models, (repo) => + modelsStore.status.getDraftSidecars(repo) + ) + ); + let allModels = $derived(modelsStore.models); - let selected = $derived(allModels.find((option) => option.id === selectedId) ?? null); + // the download groups stand in for rows the listing does not carry yet, so a + // click on them still finds its option and opens the pane + let selected = $derived( + allModels.find((option) => option.id === selectedId) ?? + downloads.find((group) => group.base.id === selectedId)?.base ?? + null + ); // The pane is laid out before it is ever opened, so the first open only slides a // finished panel in. It renders the selection, else the model it last showed. let lastPicked = $state(null); @@ -42,7 +55,11 @@ let shownId = $state(null); let isSwapping = $state(false); let fade = $state<'open' | 'swap'>('open'); - let shownOption = $derived(allModels.find((option) => option.id === shownId) ?? null); + let shownOption = $derived( + allModels.find((option) => option.id === shownId) ?? + downloads.find((group) => group.base.id === shownId)?.base ?? + null + ); $effect(() => { const id = selectedId; @@ -144,10 +161,6 @@ rank.size === 0 ? list : [...list].sort((a, b) => rankOf(a) - rankOf(b)); // one entry per repo, so a model with several quants takes a single table row let entries = $derived(byRecency(groupModelQuants(matching))); - // tracked downloads get their own section, listed like any other model - let downloads = $derived( - downloadGroups(modelsStore.status.getDownloadEntries(), modelsStore.models) - ); let groups = $derived.by(() => { // a loaded quant is a model of its own: its repo keeps the quants left behind const isLoaded = (option: ModelOption) => modelsStore.isModelLoaded(option.model); @@ -216,13 +229,16 @@ return ordered; }); - // a caller can ask for one model to be revealed, the download rows do + // a caller can ask for one model to be revealed, the download rows do; a + // tracked download is not in the listing yet, so its own rows are searched too $effect(() => { const focus = uiStore.manageModelFocus; if (!focus) return; - const option = allModels.find((model) => model.id === focus || model.model === focus); + const matches = (candidate: ModelOption) => candidate.id === focus || candidate.model === focus; + const option = + allModels.find(matches) ?? downloads.find((group) => matches(group.base))?.base ?? null; if (option) selectedId = option.id; diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerDownloadControl.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerDownloadControl.svelte index 0e81500f2c1f..5e77caed06e8 100644 --- a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerDownloadControl.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerDownloadControl.svelte @@ -38,10 +38,7 @@ class="h-5 w-5 hover:text-foreground" icon={isPaused ? MODEL_DOWNLOAD_ICONS.resume : MODEL_DOWNLOAD_ICONS.pause} iconSize="h-4 w-4" - onclick={() => - void (isPaused - ? modelsStore.status.downloadModel(option.model) - : modelsStore.status.pauseDownload(option.model))} + onclick={() => void modelsStore.status.toggleDownload(option.model)} stopPropagationOnClick tooltip={isPaused ? 'Resume downloading' : 'Pause downloading'} tooltipAsTitle diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfiguration.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfiguration.svelte index 8a109a8db41e..40a7565ed272 100644 --- a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfiguration.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfiguration.svelte @@ -1,4 +1,6 @@
- {#if isMobile} - - - {/if} + Pause downloading + {/if} + -
- {#if !isMobile} - +
+ + {#if onSeeInDiscover && discoverEnabled} + {/if} + {:else} + {#if isMobile} + + {/if} -
+ +
+ {#if !isMobile} + + {/if} + + {#if canToggleLoad} + + {/if} + + {#if onSeeInDiscover && discoverEnabled} + + {/if} +
+ {/if}
diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationHeader.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationHeader.svelte index 40f5d0d5481b..d2cc2d5b8822 100644 --- a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationHeader.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationHeader.svelte @@ -8,8 +8,14 @@ import type { ModelOption } from '$lib/types/models'; interface Props { + isDownload?: boolean; + isDownloadPaused?: boolean; isLoaded: boolean; onClose: () => void; + /** Opens the manager dialog on the Discover view, focused on this model's repo. */ + onSeeInDiscover?: () => void; + onRequestCancelDownload?: () => void; + onToggleDownload?: () => void; onToggleLoad: () => void; onUseInChat: () => void; onUseInNewChat: () => void; @@ -18,13 +24,27 @@ status: ServerModelStatus | null; } - let { isLoaded, onClose, onToggleLoad, onUseInChat, onUseInNewChat, option, status }: Props = - $props(); + let { + isDownload = false, + isDownloadPaused = false, + isLoaded, + onClose, + onRequestCancelDownload, + onSeeInDiscover, + onToggleDownload, + onToggleLoad, + onUseInChat, + onUseInNewChat, + option, + status + }: Props = $props(); let isMobile = $derived(deviceStore.isMobile); let isLoading = $derived(status === ServerModelStatus.LOADING); let statusLabel = $derived.by(() => { + if (isDownload) return isDownloadPaused ? 'Paused' : 'Downloading'; + if (status === ServerModelStatus.LOADING) return 'Loading'; if (status === ServerModelStatus.FAILED) return 'Failed to load'; @@ -37,6 +57,8 @@ }); let statusDot = $derived.by(() => { + if (isDownload) return isDownloadPaused ? 'bg-yellow-500' : 'bg-blue-500 animate-pulse'; + if (status === ServerModelStatus.FAILED) return 'bg-red-500'; if (status === ServerModelStatus.LOADING) return 'bg-muted-foreground/50 animate-pulse'; @@ -100,8 +122,13 @@ {#if !isMobile} boolean; @@ -20,11 +21,21 @@ onDelete: (option: ModelOption) => void; onSelect: (option: ModelOption) => void; selected: boolean; + /** Draft sidecars already on disk, when the row stands for a download. */ + sidecarBadges?: ModelSidecarBadge[]; /** Left padding in px, from the nesting depth. */ indent?: number; } - let { indent = 0, isFavorite, onDelete, onSelect, option, selected }: Props = $props(); + let { + indent = 0, + isFavorite, + onDelete, + onSelect, + option, + selected, + sidecarBadges + }: Props = $props(); let favorite = $derived(isFavorite(option)); let isHidden = $derived(modelsStore.isHidden(option.id)); @@ -38,6 +49,10 @@ ? ModelRowDownloadState.DOWNLOADING : null ); + + function activate(): void { + onSelect(option); + }
- - + favorited + {/if} + + + + {#if sidecarBadges?.length} + +
+ {#each sidecarBadges as badge (badge.repo + ':' + badge.kind + ':' + (badge.quant ?? ''))} + + {/each} +
+ {/if} +
(); @@ -129,6 +130,10 @@ // cancel is confirmed once for the whole list, so one dialog serves every row function requestDelete(option: ModelOption): void { pendingDelete = option.model; + // an in-flight row cancels (and drops the partials), a paused one deletes + pendingAction = modelsStore.status.isDownloadInProgress(option.model) + ? ModelDownloadConfirmAction.CANCEL + : ModelDownloadConfirmAction.DELETE; deleteOpen = true; } @@ -248,6 +253,7 @@ {onSelect} option={entry.base} selected={selectedId === entry.base.id} + sidecarBadges={entry.sidecarBadges} /> {/if} {/snippet} @@ -328,7 +334,7 @@ {/snippet} (deleteOpen = false)} open={deleteOpen} repoWithTag={pendingDelete} diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/row-actions.ts b/tools/ui/src/lib/components/app/models/ModelsManager/row-actions.ts index 9931dd3f475f..3407634ec6f1 100644 --- a/tools/ui/src/lib/components/app/models/ModelsManager/row-actions.ts +++ b/tools/ui/src/lib/components/app/models/ModelsManager/row-actions.ts @@ -1,10 +1,10 @@ -import { Eye, EyeOff, Heart, HeartOff, Trash2 } from '@lucide/svelte'; +import { Compass, Eye, EyeOff, Heart, HeartOff, Trash2 } from '@lucide/svelte'; import { MODEL_DOWNLOAD_ICONS } from '$lib/constants'; import { ModelRowDownloadState } from '$lib/enums'; -import { modelsStore } from '$lib/stores'; +import { modelsStore, uiStore } from '$lib/stores'; import type { ModelOption } from '$lib/types/models'; +import { repoOf } from '$lib/utils'; -/** Row actions follow the app's dropdown pattern: icon, label, separators, variants. */ export function modelRowActions( option: ModelOption, favorite: boolean, @@ -13,9 +13,17 @@ export function modelRowActions( /** Download state, when the row stands for a tracked download. */ download?: ModelRowDownloadState | null ) { + const viewInDiscover = { + icon: Compass, + label: 'See in Discover', + // the details pane is keyed by repo, not by `:` + onclick: () => uiStore.openModelsDiscover(repoOf(option.model)), + separator: true + }; + // a tracked download is paused, resumed or dropped: it is not loaded, hidden or - // drafted. Deleting it stops the download and removes what is on disk, which is - // what the same wording offers on a cached model + // drafted. An in-flight row cancels, which stops the download and removes the + // partial files; a paused one deletes, which removes the partials from disk if (download) { const isPaused = download === ModelRowDownloadState.PAUSED; @@ -23,18 +31,16 @@ export function modelRowActions( { icon: isPaused ? MODEL_DOWNLOAD_ICONS.resume : MODEL_DOWNLOAD_ICONS.pause, label: isPaused ? 'Resume downloading' : 'Pause downloading', - onclick: () => - void (isPaused - ? modelsStore.status.downloadModel(option.model) - : modelsStore.status.pauseDownload(option.model)) + onclick: () => void modelsStore.status.toggleDownload(option.model) }, { icon: Trash2, - label: 'Delete from disk', + label: isPaused ? 'Delete from disk' : 'Cancel download', onclick: () => onDelete(option), separator: true, variant: 'destructive' as const - } + }, + viewInDiscover ]; } @@ -56,6 +62,7 @@ export function modelRowActions( label: isHidden ? 'Unhide model' : 'Hide model', onclick: () => modelsStore.toggleHidden(option.id), separator: true - } + }, + viewInDiscover ]; } diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/utils.ts b/tools/ui/src/lib/components/app/models/ModelsManager/utils.ts index 208343664942..435700a10394 100644 --- a/tools/ui/src/lib/components/app/models/ModelsManager/utils.ts +++ b/tools/ui/src/lib/components/app/models/ModelsManager/utils.ts @@ -2,7 +2,12 @@ import { LOCAL_BACKEND_ID, type ModalityKey } from '$lib/constants'; import { ModelCapability, ModelGroupKind, ModelsTableGroupKind } from '$lib/enums'; import { HuggingFaceService, ModelsService } from '$lib/services'; import { modelsStore } from '$lib/stores'; -import type { ModelDownloadEntry, ModelDownloadProgress, ModelOption } from '$lib/types/models'; +import type { + ModelDownloadEntry, + ModelDownloadProgress, + ModelOption, + ModelSidecarBadge +} from '$lib/types/models'; import { detectThinkingSupport, detectToolUseSupport, repoOf } from '$lib/utils'; /** One repo of the table, with the rows it ships as. */ @@ -11,6 +16,8 @@ export interface ModelQuantGroup { key: string; kind: ModelGroupKind; quants: ModelOption[]; + /** Draft sidecars already on disk, shown as checked sub-lines. */ + sidecarBadges?: ModelSidecarBadge[]; } /** True when any of the manager's filters narrows the table. */ @@ -31,28 +38,36 @@ export function downloadProgressFor(repoWithTag: string): ModelDownloadProgress } /** - * One table row per tracked download, so the table lists them the way it lists - * any other model. A paused download is already in the router's listing, so its - * own option is reused; a fresh one stands for its tag alone. + * One table row per tracked download, listed like any other model. A paused download + * is already in the router's listing, so its own option is reused; a fresh one is + * synthesized for its tag alone. */ export function downloadGroups( entries: ModelDownloadEntry[], - models: ModelOption[] + models: ModelOption[], + sidecarsFor: (repo: string) => ModelSidecarBadge[] ): ModelQuantGroup[] { return entries.map((entry) => { - const option = models.find((candidate) => candidate.model === entry.repoWithTag) ?? { + const listed = models.find((candidate) => candidate.model === entry.repoWithTag); + // the synthesized option carries backendId, which ModelOption drops on + // assignment, so the callers spread it instead of typing this as one + const option = listed ?? { backendId: LOCAL_BACKEND_ID, capabilities: [], id: entry.repoWithTag, model: entry.repoWithTag, name: entry.repoWithTag }; + // a draft still coming down badges the id, an on-disk one keeps its sub-line + const sidecars = sidecarsFor(repoOf(entry.repoWithTag)); + const pending = sidecars.filter((badge) => badge.pending); return { - base: option, + base: { ...option, draftSidecars: pending }, key: `download::${entry.repoWithTag}`, kind: ModelGroupKind.QUANTS, - quants: [option] + quants: [{ ...option, draftSidecars: pending }], + sidecarBadges: sidecars.filter((badge) => !badge.pending) }; }); } diff --git a/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDownloadItem.svelte b/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDownloadItem.svelte index 6ef1433b970f..7cc286f89ddf 100644 --- a/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDownloadItem.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDownloadItem.svelte @@ -1,22 +1,38 @@ - -
- - + when paused; opacity only for the spinner, the spin owns the transform --> + - - - onRequestCancel?.(entry.repoWithTag)} - /> + + + + + // a phone has no manager: its information dialog takes the icon + deviceStore.isMobile + ? uiStore.openModelInformation(option) + : uiStore.openModelsManager(entry.repoWithTag)} + tooltip="Manage model" + tooltipAsTitle + /> + + onRequestCancel?.(entry.repoWithTag)} + /> +
+ + + {#if onDiskDrafts.length} +
+ {#each onDiskDrafts as badge (badge.repo + ':' + badge.kind + ':' + (badge.quant ?? ''))} + + {/each} +
+ {/if} {#if entry.progress && entry.progress.totalBytes > 0} Downloading

{#each downloadEntries as entry (entry.repoWithTag)} - + {/each} {/if} @@ -230,6 +236,7 @@ (cancelOpen = false)} + onConfirm={() => void modelsStore.status.cancelDownload(pendingCancel)} open={cancelOpen} repoWithTag={pendingCancel} /> diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscover.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscover.svelte new file mode 100644 index 000000000000..292938d59c5c --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscover.svelte @@ -0,0 +1,210 @@ + + +{#snippet catalog()} + (selectedId = id)} + showBaseModelAvatar + /> +{/snippet} + + + +
+ {#if selectedId} + + {/if} +
+ + (cancelOpen = false)} + onConfirm={() => void modelsStore.status.cancelDownload(pendingCancel)} + open={cancelOpen} + repoWithTag={pendingCancel} +/> diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverChatTemplateDialog.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverChatTemplateDialog.svelte new file mode 100644 index 000000000000..11240a4a91e1 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverChatTemplateDialog.svelte @@ -0,0 +1,75 @@ + + + + + + + Chat template + + + + + + + + + + + diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetails.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetails.svelte new file mode 100644 index 000000000000..8f7e50da6c24 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetails.svelte @@ -0,0 +1,96 @@ + + +{#if loading} + +{:else if error} +
+

{error}

+
+{:else if details} +
+ + + + + +
+{/if} diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsContainer.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsContainer.svelte new file mode 100644 index 000000000000..7d81ea8abab9 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsContainer.svelte @@ -0,0 +1,66 @@ + + + diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptions.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptions.svelte new file mode 100644 index 000000000000..a1bd05a37eb2 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptions.svelte @@ -0,0 +1,258 @@ + + +{#if bitDepthRows.length} +
+ +
+ {#each rows as row (row.bitDepth)} + (draftPick = kind)} + onRequestCancel={requestCancel} + onRequestDelete={requestDelete} + /> + {/each} +
+ + + {#if draftKinds.length} +
+ + + value && (draftPick = value as ModelDraftSidecar)} + type="single" + value={draftKind} + > + + {draftKindLabel} + + + + {#each draftKinds as kind (kind)} + + {kind.toUpperCase()} + + {/each} + + + + +
+ {/if} +
+{/if} + + (confirmOpen = false)} + open={confirmOpen} + repoWithTag={pending.repoWithTag} +/> diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsDownloadCommand.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsDownloadCommand.svelte new file mode 100644 index 000000000000..eb7f4e55f43b --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsDownloadCommand.svelte @@ -0,0 +1,131 @@ + + +
+ +
+ {SERVE_COMMAND.BIN} + + {SERVE_COMMAND.SUBCOMMAND} + + {SERVE_COMMAND.MODEL_FLAG} + + + {modelId}{quant ? MODEL_ID.QUANTIZATION_SEPARATOR : ''} + + + + {#if quant} + + {quant} + + {/if} + + + {#if draftKind} + {SERVE_COMMAND.SPEC_TYPE_FLAG} + + + {#if draftKinds.length > 1} + value && onDraftKindChange?.(value as ModelDraftSidecar)} + type="single" + value={draftKind} + > + + {SPEC_TYPE[draftKind]} + + + + {#each draftKinds as kind (kind)} + + {SPEC_TYPE[kind]} + + {/each} + + + {:else} + + {SPEC_TYPE[draftKind]} + + {/if} + {/if} +
+ + +
diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsDraftDownloadButton.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsDraftDownloadButton.svelte new file mode 100644 index 000000000000..d7cbd8becd84 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsDraftDownloadButton.svelte @@ -0,0 +1,61 @@ + + + + + + {#snippet child({ props })} + + {/snippet} + + + +

{tooltipText}

+ + diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsQuantDownloadButton.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsQuantDownloadButton.svelte new file mode 100644 index 000000000000..5ccbd31d08c7 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsQuantDownloadButton.svelte @@ -0,0 +1,272 @@ + + +{#snippet chipBody(dividerClass: string)} + + {label} + + {#if showsCompanion && companion} + + + + + + {companion.kind}{companion.quant ? `:${companion.quant}` : ''} + + {#if companion.onDisk} + + {/if} + + {/if} + + +{/snippet} + + +{#snippet tooltipTrigger(tip: string, button: Snippet<[Record]>)} + + + {#snippet child({ props })} + {@render button(props)} + {/snippet} + + + +

{tip}

+ + +{/snippet} + +{#snippet deleteChip(props: Record)} + + +{/snippet} + +{#snippet pauseResumeChip(props: Record)} + +{/snippet} + +{#snippet cancelChip(props: Record)} + + +{/snippet} + +{#snippet downloadChip(props: Record)} + + +{/snippet} + +{#if entry.isDownloaded} + {@render tooltipTrigger(tooltipText, deleteChip)} +{:else if entry.isDownloading || entry.isPaused} + +
+ {@render tooltipTrigger(tooltipText, pauseResumeChip)} + + {@render tooltipTrigger('Cancel downloading', cancelChip)} + + {#if percent !== null} + + {/if} +
+{:else} + {@render tooltipTrigger(tooltipText, downloadChip)} +{/if} diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsRow.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsRow.svelte new file mode 100644 index 000000000000..4527014fc901 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsRow.svelte @@ -0,0 +1,134 @@ + + +
+
+ + + +
+ {#each files as file (file.path)} + + {/each} + + {#each missingDrafts as draft (draft.repoWithTag + ':' + draft.companion.kind)} + + {/each} +
+
+ + {#if commandOpen} +
+ +
+ {/if} +
diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/download-options.utils.ts b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/download-options.utils.ts new file mode 100644 index 000000000000..3c07cbafdf70 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/download-options.utils.ts @@ -0,0 +1,57 @@ +import { DRAFT_FILE_LABEL, MODEL_ID, PATH_SEPARATOR } from '$lib/constants'; +import { ModelDraftSidecar, ModelSelectableFileKind } from '$lib/enums'; +import { HuggingFaceService } from '$lib/services'; +import type { ModelSelectableFile } from '$lib/types'; +import { isAuxSidecar, isDraftSidecar } from '$lib/utils'; + +/** + * Draft kinds in the order a chip lists them. DFlash wins: on the models that + * ship both it drafts a block at a time and stays accurate further out than + * MTP's few tokens; a kind not listed here sorts last. + */ +export const DRAFT_PREFERENCE_ORDER: readonly ModelDraftSidecar[] = [ + ModelDraftSidecar.DFLASH, + ModelDraftSidecar.MTP +]; + +/** Draft sidecar of a file, typed for files already known to be drafts; null when it carries none. */ +export function draftBadgeOf(file: Pick): ModelDraftSidecar | null { + const sidecar = HuggingFaceService.extractQuantMeta(file.path)?.sidecar; + + return sidecar && isDraftSidecar(sidecar) ? sidecar : null; +} + +/** Kind of a file path: the main weights, a draft sidecar, or an aux sidecar (mmproj). */ +export function classify(path: string): ModelSelectableFileKind { + const sidecar = HuggingFaceService.extractQuantMeta(path)?.sidecar; + + if (!sidecar) return ModelSelectableFileKind.MAIN; + + return isAuxSidecar(sidecar) ? ModelSelectableFileKind.AUX : ModelSelectableFileKind.DRAFT; +} + +/** Draft kinds best first, so a chip lists its joined badges in the pull order. */ +export function sortDraftKinds(kinds: readonly ModelDraftSidecar[]): ModelDraftSidecar[] { + const rank = (kind: ModelDraftSidecar) => { + const index = DRAFT_PREFERENCE_ORDER.indexOf(kind); + + return index === -1 ? DRAFT_PREFERENCE_ORDER.length : index; + }; + + return [...kinds].sort((a, b) => rank(a) - rank(b)); +} + +/** Display label of a file: its quant, else the file name without the extension. */ +export function labelFor(path: string): string { + const meta = HuggingFaceService.extractQuantMeta(path); + + if (meta?.quant) return meta.quant; + + // Quantless sidecar files: the chip badge already carries the sidecar type, + // so the label only marks draft files; aux sidecars badge alone. + if (meta?.sidecar) return isDraftSidecar(meta.sidecar) ? DRAFT_FILE_LABEL : ''; + + const basename = path.split(PATH_SEPARATOR).pop() ?? path; + + return basename.replace(MODEL_ID.WEIGHT_EXTENSION_REGEX, ''); +} diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/index.ts b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/index.ts new file mode 100644 index 000000000000..bb82218ccb4b --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/index.ts @@ -0,0 +1,24 @@ +/** + * + * MODELS DISCOVER - DETAILS - DOWNLOAD OPTIONS + * + * The download area of the detail pane: one chip per main quant, the drafts it rides with + * badged on it, a draft checkbox and type pick at the bottom, and a terminal command + * preview inside every row. + * + */ + +/** One download action per main quant, the sidecars it pulls, and the row's command. */ +export { default as ModelsDiscoverDetailsDownloadOptions } from './ModelsDiscoverDetailsDownloadOptions.svelte'; + +/** One bit-depth row: the depth label opens its command, the quants of that depth download beside it. */ +export { default as ModelsDiscoverDetailsDownloadOptionsRow } from './ModelsDiscoverDetailsDownloadOptionsRow.svelte'; + +/** A single GGUF file as an independent action chip. */ +export { default as ModelsDiscoverDetailsDownloadOptionsQuantDownloadButton } from './ModelsDiscoverDetailsDownloadOptionsQuantDownloadButton.svelte'; + +/** Chip for a draft a downloaded model is missing; posts the model with that kind, so the server keeps what is on disk and pulls the draft. */ +export { default as ModelsDiscoverDetailsDownloadOptionsDraftDownloadButton } from './ModelsDiscoverDetailsDownloadOptionsDraftDownloadButton.svelte'; + +/** The `llama serve -hf ...` command box of one row, with a draft type pick and a copy button. */ +export { default as ModelsDiscoverDetailsDownloadOptionsDownloadCommand } from './ModelsDiscoverDetailsDownloadOptionsDownloadCommand.svelte'; diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsHeader.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsHeader.svelte new file mode 100644 index 000000000000..8687650796b9 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsHeader.svelte @@ -0,0 +1,96 @@ + + +
+
+
+ + +
+
+

{details.id ?? modelId}

+ + +
+ + {#if baseModels.length} +
+ {baseModels.join(', ')} + + + + +
+ {/if} +
+
+ + +
+ + + + +
diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsHfHubStats.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsHfHubStats.svelte new file mode 100644 index 000000000000..b848a9bfeb35 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsHfHubStats.svelte @@ -0,0 +1,31 @@ + + +
+ {#if typeof details.downloads === 'number'} + + + {HuggingFaceService.formatDownloads(details.downloads)} + + {/if} + + {#if typeof details.likes === 'number'} + + + {HuggingFaceService.formatLikes(details.likes)} + + {/if} + + {#if details.lastModified} + Updated {HuggingFaceService.formatRelativeTime(details.lastModified)} + {/if} +
diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsMetadata.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsMetadata.svelte new file mode 100644 index 000000000000..af7a38089390 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsMetadata.svelte @@ -0,0 +1,82 @@ + + +{#if description} +

{description}

+{/if} + + +
+ {#if gguf?.total} + + {/if} + + {#if gguf?.context_length} + + {/if} + + {#if gguf?.architecture} + + {/if} + + {#if gguf?.chat_template} + + {/if} + + {#if licenseTag} + + {/if} + + {#if details.gated === true} + + gated + + {/if} +
+ +{#if gguf?.chat_template} + +{/if} diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsMetadataItem.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsMetadataItem.svelte new file mode 100644 index 000000000000..4bfc027d8e1e --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsMetadataItem.svelte @@ -0,0 +1,18 @@ + + + + + {label} + + {value} + diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsReadme.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsReadme.svelte new file mode 100644 index 000000000000..fbf043b79712 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsReadme.svelte @@ -0,0 +1,17 @@ + + +{#if readme} +
+ +
+{/if} diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsSkeleton.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsSkeleton.svelte new file mode 100644 index 000000000000..8438a3c2b47d --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsSkeleton.svelte @@ -0,0 +1,98 @@ + + + +
+ +
+
+
+ + +
+ + + +
+
+ + +
+ + +
+ + + + + +
+ + + + + +
+
+ + + + + + + +
+
+
+ + +
+
+ {#each [0, 1, 2] as _, index (index)} +
+
+ + + +
+ +
+ + + +
+
+ {/each} +
+ +
+ + +
+ + + + + +
+ + +
+
+ + +
+
+ + + + + + + +
+
+
diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/index.ts b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/index.ts new file mode 100644 index 000000000000..91d5c89065d6 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/index.ts @@ -0,0 +1,37 @@ +/** + * + * MODELS DISCOVER - DETAILS + * + * The right-hand detail pane of the discover view: header (avatar, name, stats, metadata + * chips, capability badges), the download options area and the model-card README. + * + */ + +/** Detail pane of the selected model; fetches its own details and GGUF file list. */ +export { default as ModelsDiscoverDetails } from './ModelsDiscoverDetails.svelte'; + +/** Fetches one repo's details, files and readme for hosts outside Discover. */ +export { default as ModelsDiscoverDetailsContainer } from './ModelsDiscoverDetailsContainer.svelte'; + +/** Header of the detail pane. */ +export { default as ModelsDiscoverDetailsHeader } from './ModelsDiscoverDetailsHeader.svelte'; + +/** Downloads, likes and last updated, formatted the way the Hub shows them. */ +export { default as ModelsDiscoverDetailsHfHubStats } from './ModelsDiscoverDetailsHfHubStats.svelte'; + +/** Static placeholder matching the detail layout. */ +export { default as ModelsDiscoverDetailsSkeleton } from './ModelsDiscoverDetailsSkeleton.svelte'; + +/** Model card README as markdown. */ +export { default as ModelsDiscoverDetailsReadme } from './ModelsDiscoverDetailsReadme.svelte'; + +/** Metadata chips: size, context, architecture, license, chat template. */ +export { default as ModelsDiscoverDetailsMetadata } from './ModelsDiscoverDetailsMetadata.svelte'; + +/** One label | value chip of the metadata row. */ +export { default as ModelsDiscoverDetailsMetadataItem } from './ModelsDiscoverDetailsMetadataItem.svelte'; + +/** Chat template in a scrollable dialog with a copy button. */ +export { default as ModelsDiscoverChatTemplateDialog } from './ModelsDiscoverChatTemplateDialog.svelte'; + +export * from './ModelsDiscoverDetailsDownloadOptions'; diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverList.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverList.svelte new file mode 100644 index 000000000000..5ebff7230d32 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverList.svelte @@ -0,0 +1,41 @@ + + +
    + {#if loading} + {#each Array(loadingSkeletonRowsCount) as _, index (index)} + + {/each} + {:else} + {#each models as model (model.id)} + + {/each} + {/if} +
diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverListItem.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverListItem.svelte new file mode 100644 index 000000000000..1eee85ea0777 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverListItem.svelte @@ -0,0 +1,256 @@ + + +
  • +
    + + + {#if download} + {@const state = download} + + + + + {percent !== null ? `${percent}%` : state.isPaused ? 'Paused' : 'Downloading'} + + + + + {/if} +
    + + + {#if onDiskDrafts.length} +
    + {#each onDiskDrafts as badge (badge.repo + ':' + badge.kind + ':' + (badge.quant ?? ''))} + + {/each} +
    + {/if} + + {#if download?.progress && download.progress.totalBytes > 0} + + {/if} +
    • diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverListItemSkeleton.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverListItemSkeleton.svelte new file mode 100644 index 000000000000..851abc523860 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverListItemSkeleton.svelte @@ -0,0 +1,39 @@ + + + +
  • +
    + + +
    + + +
    + {#each badgeWidths as width, i (i)} + + {/each} +
    +
    +
    +
    • diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverListSearch.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverListSearch.svelte new file mode 100644 index 000000000000..458c31648c07 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverListSearch.svelte @@ -0,0 +1,30 @@ + + + +
    + handleInput(v)} {placeholder} size="sm" /> +
    diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/index.ts b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/index.ts new file mode 100644 index 000000000000..c3025c0b2601 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/index.ts @@ -0,0 +1,16 @@ +/** + * + * MODELS DISCOVER - LIST + * + * The discover sidebar column: a debounced search field above a navigable list of + * HuggingFace GGUF models, with skeleton rows while loading. + * + */ + +export { default as ModelsDiscoverList } from './ModelsDiscoverList.svelte'; +export { default as ModelsDiscoverListSearch } from './ModelsDiscoverListSearch.svelte'; + +/** One model entry; also the row of a tracked download, with its own controls. */ +export { default as ModelsDiscoverListItem } from './ModelsDiscoverListItem.svelte'; + +export { default as ModelsDiscoverListItemSkeleton } from './ModelsDiscoverListItemSkeleton.svelte'; diff --git a/tools/ui/src/lib/components/app/models/discover/index.ts b/tools/ui/src/lib/components/app/models/discover/index.ts new file mode 100644 index 000000000000..bb353c72dfa8 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/index.ts @@ -0,0 +1,16 @@ +/** + * + * MODELS DISCOVER + * + * Components for the Models Discover view: a sidebar search + list of HuggingFace GGUF + * models and a detail view for the selected model, used as the body of the discovery + * dialog. The list and detail trees live in their own subfolders; this barrel re-exports + * them alongside the shared leaves. + * + */ + +/** The whole view: sidebar list on the left, detail pane on the right. */ +export { default as ModelsDiscover } from './ModelsDiscover.svelte'; + +export * from './ModelsDiscoverList'; +export * from './ModelsDiscoverDetails'; diff --git a/tools/ui/src/lib/components/app/models/index.ts b/tools/ui/src/lib/components/app/models/index.ts index dbfb5c60e248..83f076fd11b0 100644 --- a/tools/ui/src/lib/components/app/models/index.ts +++ b/tools/ui/src/lib/components/app/models/index.ts @@ -59,4 +59,7 @@ export { default as ModelCapabilityIcons } from './ModelCapabilityIcons.svelte'; /** Draft sidecar badges, rendered as `+ [KIND] [QUANT]` per sidecar. */ export { default as ModelDraftSidecars } from './ModelDraftSidecars.svelte'; + +/** Compact draft download line, embedded in the item it belongs to. */ +export { default as ModelDraftSubline } from './ModelDraftSubline.svelte'; export * from './ModelsSelector'; diff --git a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte index 879390075988..aebf37251384 100644 --- a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte +++ b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte @@ -13,6 +13,7 @@ NUMERIC_FIELDS, POSITIVE_INTEGER_FIELDS, SETTINGS_CHAT_SECTIONS, + SETTINGS_KEYS, SETTINGS_SECTION_SLUGS } from '$lib/constants'; import { ColorMode } from '$lib/enums/ui.enums'; @@ -70,6 +71,12 @@ function handleConfigChange(key: string, value: string | boolean) { localConfig[key] = value; + + // discover needs the hub metadata reads: checking it turns the hub on, + // unchecking it turns the hub back off, so the checkbox never lies + if (key === SETTINGS_KEYS.ENABLE_DISCOVER_MODELS) { + localConfig[SETTINGS_KEYS.USE_HUGGING_FACE_HUB] = value === true; + } } function handleReset() { diff --git a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatFields.svelte b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatFields.svelte index e8795d7c5204..5dfba6bb374c 100644 --- a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatFields.svelte +++ b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatFields.svelte @@ -22,6 +22,13 @@ let { fields, localConfig, onConfigChange, onThemeChange }: Props = $props(); + // discover needs the hub metadata reads, so its checkbox is read-only while discover is on + let hubLocked = $derived(Boolean(localConfig[SETTINGS_KEYS.ENABLE_DISCOVER_MODELS])); + + function isLockedField(key: string): boolean { + return hubLocked && key === SETTINGS_KEYS.USE_HUGGING_FACE_HUB; + } + let currentModelParams = $derived.by(() => { void modelsStore.props.cacheVersion; @@ -301,13 +308,18 @@ onConfigChange(field.key, checked)} />