From 57e3ffd973a750df22b0df1572c63f0e535a9fd9 Mon Sep 17 00:00:00 2001 From: Anubhav Srivastava Date: Tue, 28 Jul 2026 00:49:30 +0530 Subject: [PATCH 1/4] q6_k GEMM working --- ggml/src/ggml-cpu/arch/arm/repack.cpp | 470 +++++++++++++++++++++++++- 1 file changed, 468 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cpp index a7534443091f..854bc67d5b9e 100644 --- a/ggml/src/ggml-cpu/arch/arm/repack.cpp +++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp @@ -14,6 +14,7 @@ #include #include // for qsort #include // for GGML_ASSERT +#include #define GGML_CPU_CLANG_WORKAROUND #include "../../repack.h" @@ -4725,6 +4726,7 @@ void ggml_gemm_q6_K_8x8_q8_K(int n, const void * GGML_RESTRICT vy, int nr, int nc) { + // std::cout<<"ggml_gemm_q6_K_8x8_q8_K called"; constexpr int qk = QK_K; const int nb = n / qk; @@ -4738,9 +4740,473 @@ void ggml_gemm_q6_K_8x8_q8_K(int n, UNUSED(nb); UNUSED(ncols_interleaved); UNUSED(blocklen); + constexpr int q8_k_blocklen = 4; + +#if defined(__aarch64__) && defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8) + switch(svcntb() * 8){ + // case 256:{ + // // std::cout<<"SVE 256 called"; + // constexpr int col_pairs = ncols_interleaved / 2; + // const svuint8_t m4b = svdup_n_u8(0x0f); + // const svuint8_t mask_lo = svdup_n_u8(0x03); + // const svuint8_t mask_hi = svdup_n_u8(0x30); + // svint32_t zeros = svdup_s32(0); + // svfloat32_t zeros_fp32 = svdup_f32(0.0f); + // svfloat16_t fp16_zero = svdup_n_f16((__fp16)0.0); + // svbool_t pg32_2 = svwhilelt_b32(0, 2); + // svbool_t pg32_4 = svwhilelt_b32(0, 4); + // svbool_t pg32_8 = svptrue_b32(); + // svbool_t pg16_4 = svwhilelt_b16(0, 4); + // svbool_t pg16_8 = svwhilelt_b16(0, 8); + // svbool_t pg8_32 = svptrue_b8(); + // svbool_t ph = svptrue_pat_b8(SV_VL16); + // svbool_t pl = svnot_b_z(svptrue_b8(), ph); + + // // 8 accumulators: 2 row pairs × 4 col pairs + // svfloat32_t acc_f32_01, acc_f32_23, acc_f32_45, acc_f32_67; + // uint32_t idx_arr[8] = { 0, 2, 4, 6, 1, 3, 5, 7 }; + // svbool_t pg = svptrue_pat_b32(SV_VL8); + // svuint32_t idx = svld1(pg, idx_arr); + + // static const uint32_t idx_data[8] = {0, 4, 2, 6, 1, 5, 3, 7}; + // svuint32_t idx1 = svld1_u32(svptrue_b32(), idx_data); + + // for (int y = 0; y < nr / q8_k_blocklen; y++) { + // const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + (y * nb); + + // for (int x = 0; x < nc / ncols_interleaved; x++) { + // const block_q6_Kx8 * GGML_RESTRICT q6_ptr = (const block_q6_Kx8 *) vx + (x * nb); + + // acc_f32_01 = svdup_n_f32(0); + // acc_f32_23 = svdup_n_f32(0); + // acc_f32_45 = svdup_n_f32(0); + // acc_f32_67 = svdup_n_f32(0); + + // for (int b = 0; b < nb; b++) { + // svint32_t sb_acc_0 = svdup_n_s32(0); + // svint32_t sb_acc_2 = svdup_n_s32(0); + + // svint32_t acc_00 = svdup_n_s32(0); + // svint32_t acc_11 = svdup_n_s32(0); + // svint32_t acc_22 = svdup_n_s32(0); + // svint32_t acc_33 = svdup_n_s32(0); + // svint32_t acc_44 = svdup_n_s32(0); + // svint32_t acc_55 = svdup_n_s32(0); + // svint32_t acc_66 = svdup_n_s32(0); + // svint32_t acc_77 = svdup_n_s32(0); + + // for (int sb = 0; sb < QK_K / 64; sb++) { + // const int8_t * q8_base_1 = q8_ptr[b].qs + sb * 256; + + // svint8_t q8_qs_0 = svadd_s8_x(svptrue_b8(), svld1_s8(ph, q8_base_1 + 0), svld1_s8(pl, q8_base_1 + 112)); + // svint8_t q8_qs_2 = svadd_s8_x(svptrue_b8(), svld1_s8(ph, q8_base_1 + 64), svld1_s8(pl, q8_base_1 + 144)); + // svint8_t q8_qs_4 = svadd_s8_x(svptrue_b8(), svld1_s8(ph, q8_base_1 + 128), svld1_s8(pl, q8_base_1 + 176)); + // svint8_t q8_qs_6 = svadd_s8_x(svptrue_b8(), svld1_s8(ph, q8_base_1 + 192), svld1_s8(pl, q8_base_1 + 208)); + + // svint8_t q8_qs_1 = svadd_s8_x(svptrue_b8(), svld1_s8(ph, q8_base_1 + 16), svld1_s8(pl, q8_base_1 + 128)); + // svint8_t q8_qs_3 = svadd_s8_x(svptrue_b8(), svld1_s8(ph, q8_base_1 + 80), svld1_s8(pl, q8_base_1 + 160)); + // svint8_t q8_qs_5 = svadd_s8_x(svptrue_b8(), svld1_s8(ph, q8_base_1 + 144), svld1_s8(pl, q8_base_1 + 192)); + // svint8_t q8_qs_7 = svadd_s8_x(svptrue_b8(), svld1_s8(ph, q8_base_1 + 208), svld1_s8(pl, q8_base_1 + 224)); + + // const int8_t *base_scale_ptr = q6_ptr[b].scales+(sb/2)*64+(sb%2)*16; + + // // Q4s columns iterated in pairs (01, 23, 45, 67) + // for (int cp = 0; cp < ncols_interleaved / 2; cp++) { + // svint32_t sb_acc_0_s0 = svdup_n_s32(0); + // svint32_t sb_acc_0_s1 = svdup_n_s32(0); + // svint32_t sb_acc_2_s0 = svdup_n_s32(0); + // svint32_t sb_acc_2_s1 = svdup_n_s32(0); + + // svuint8_t q6_qs_cp_00_l = svld1rq_u8(svptrue_b8(), q6_ptr[b].ql + sb * QK_K + 16 * cp + 0); + // svuint8_t q6_qs_cp_01_l = svld1rq_u8(svptrue_b8(), q6_ptr[b].ql + sb * QK_K + 16 * cp + 64); + // svuint8_t q6_qs_cp_02_l = svld1rq_u8(svptrue_b8(), q6_ptr[b].ql + sb * QK_K + 16 * cp + 128); + // svuint8_t q6_qs_cp_03_l = svld1rq_u8(svptrue_b8(), q6_ptr[b].ql + sb * QK_K + 16 * cp + 192); + + // // we divide by 2 because for sb=0, range is [0 to 255] + // // but for sb=2, range is [256, 511] + + // svuint8_t q6_qs_cp_00_h = svld1rq_u8(svptrue_b8(), q6_ptr[b].qh + (sb * QK_K)/2 + 16 * cp + 0); + // svuint8_t q6_qs_cp_01_h = svld1rq_u8(svptrue_b8(), q6_ptr[b].qh + (sb * QK_K)/2 + 16 * cp + 64); + // svuint8_t q6_qs_cp_02_h = svld1rq_u8(svptrue_b8(), q6_ptr[b].qh + (sb * QK_K)/2 + 16 * cp + 128); + // svuint8_t q6_qs_cp_03_h = svld1rq_u8(svptrue_b8(), q6_ptr[b].qh + (sb * QK_K)/2 + 16 * cp + 192); + + // //shift qh by 2 bits if sb == 1 or 3 since those indexes hold qh values at offset 64 + // if(sb==1||sb==3){ + // q6_qs_cp_00_h = svlsr_n_u8_x(pg8_32,q6_qs_cp_00_h, 2); + // q6_qs_cp_01_h = svlsr_n_u8_x(pg8_32,q6_qs_cp_01_h, 2); + // q6_qs_cp_02_h = svlsr_n_u8_x(pg8_32,q6_qs_cp_02_h, 2); + // q6_qs_cp_03_h = svlsr_n_u8_x(pg8_32,q6_qs_cp_03_h, 2); + // } + + // //Create the entire q values from ql and qh + // // svuint8_t ql = svlsr_n_u8_m(pl, svand_u8_m(ph, q6_qs_cp_00_l, m4b), 4); + // // svuint8_t qh = svand_u8_m(pl, svlsl_n_u8_m(ph, svand_u8_m(ph, q6_qs_cp_00_h, mask_lo), 4), mask_hi); + // // svuint8_t q = svreinterpret_s8_u8(svorr_u8_x(pg8_32, ql, qh)); + + // svint8_t q6_qs_cp_00 = svreinterpret_s8_u8( + // svorr_u8_x(pg8_32, + // svlsr_n_u8_m(pl, svand_u8_m(ph, q6_qs_cp_00_l, m4b), 4), //ql part + // svand_u8_m(pl, svlsl_n_u8_m(ph, svand_u8_m(ph, q6_qs_cp_00_h, mask_lo), 4), mask_hi) //qh part + // )); + + // svint8_t q6_qs_cp_01 = svreinterpret_s8_u8( + // svorr_u8_x(pg8_32, + // svlsr_n_u8_m(pl, svand_u8_m(ph, q6_qs_cp_01_l, m4b), 4), //ql part + // svand_u8_m(pl, svlsl_n_u8_m(ph, svand_u8_m(ph, q6_qs_cp_01_h, mask_lo), 4), mask_hi) //qh part + // )); + + // svint8_t q6_qs_cp_02 = svreinterpret_s8_u8( + // svorr_u8_x(pg8_32, + // svlsr_n_u8_m(pl, svand_u8_m(ph, q6_qs_cp_02_l, m4b), 4), //ql part + // svand_u8_m(pl, svlsl_n_u8_m(ph, svand_u8_m(ph, q6_qs_cp_02_h, mask_lo), 4), mask_hi) //qh part + // )); + + // svint8_t q6_qs_cp_03 = svreinterpret_s8_u8( + // svorr_u8_x(pg8_32, + // svlsr_n_u8_m(pl, svand_u8_m(ph, q6_qs_cp_03_l, m4b), 4), //ql part + // svand_u8_m(pl, svlsl_n_u8_m(ph, svand_u8_m(ph, q6_qs_cp_03_h, mask_lo), 4), mask_hi) //qh part + // )); + + // //Signed 32 correction + // q6_qs_cp_00 = svsub_n_s8_x(pg8_32, q6_qs_cp_00, 32); + // q6_qs_cp_01 = svsub_n_s8_x(pg8_32, q6_qs_cp_01, 32); + // q6_qs_cp_02 = svsub_n_s8_x(pg8_32, q6_qs_cp_02, 32); + // q6_qs_cp_03 = svsub_n_s8_x(pg8_32, q6_qs_cp_03, 32); + + // sb_acc_0_s0 = svmmla_s32(sb_acc_0_s0, q6_qs_cp_00, q8_qs_0); + // sb_acc_0_s0 = svmmla_s32(sb_acc_0_s0, q6_qs_cp_01, q8_qs_2); + + // sb_acc_0_s1 = svmmla_s32(sb_acc_0_s1, q6_qs_cp_02, q8_qs_4); + // sb_acc_0_s1 = svmmla_s32(sb_acc_0_s1, q6_qs_cp_03, q8_qs_6); + + // sb_acc_2_s0 = svmmla_s32(sb_acc_2_s0, q6_qs_cp_00, q8_qs_1); + // sb_acc_2_s0 = svmmla_s32(sb_acc_2_s0, q6_qs_cp_01, q8_qs_3); + + // sb_acc_2_s1 = svmmla_s32(sb_acc_2_s1, q6_qs_cp_02, q8_qs_5); + // sb_acc_2_s1 = svmmla_s32(sb_acc_2_s1, q6_qs_cp_03, q8_qs_7); + + // const int scale_col = 2 * cp; + + // int32_t scale_arr_0[8] = { + // //Low lanes + // base_scale_ptr[scale_col + 0], + // base_scale_ptr[scale_col + 0], + // base_scale_ptr[scale_col + 1], + // base_scale_ptr[scale_col + 1], + // //High Lanes + // base_scale_ptr[scale_col + 32], + // base_scale_ptr[scale_col + 32], + // base_scale_ptr[scale_col + 33], + // base_scale_ptr[scale_col + 33], + // }; + + // int32_t scale_arr_1[8] = { + // //Low lanes + // base_scale_ptr[scale_col + 8], + // base_scale_ptr[scale_col + 8], + // base_scale_ptr[scale_col + 9], + // base_scale_ptr[scale_col + 9], + // //High Lanes + // base_scale_ptr[scale_col + 40], + // base_scale_ptr[scale_col + 40], + // base_scale_ptr[scale_col + 41], + // base_scale_ptr[scale_col + 41], + // }; + + // const svint32_t scale_vec_0 = + // svld1_s32(pg32_8, scale_arr_0); + + // const svint32_t scale_vec_1 = + // svld1_s32(pg32_8, scale_arr_1); + + // if(cp == 0) { + // acc_00 = svmla_s32_m(svptrue_b32(), acc_00, sb_acc_0_s0, scale_vec_0); + // acc_00 = svmla_s32_m(svptrue_b32(), acc_00, sb_acc_0_s1, scale_vec_1); + // acc_44 = svmla_s32_m(svptrue_b32(), acc_44, sb_acc_2_s0, scale_vec_0); + // acc_44 = svmla_s32_m(svptrue_b32(), acc_44, sb_acc_2_s1, scale_vec_1); + // } + // if(cp == 1) { + // acc_11 = svmla_s32_m(svptrue_b32(), acc_11, sb_acc_0_s0, scale_vec_0); + // acc_11 = svmla_s32_m(svptrue_b32(), acc_11, sb_acc_0_s1, scale_vec_1); + // acc_55 = svmla_s32_m(svptrue_b32(), acc_55, sb_acc_2_s0, scale_vec_0); + // acc_55 = svmla_s32_m(svptrue_b32(), acc_55, sb_acc_2_s1, scale_vec_1); + // } + // if(cp == 2) { + // acc_22 = svmla_s32_m(svptrue_b32(), acc_22, sb_acc_0_s0, scale_vec_0); + // acc_22 = svmla_s32_m(svptrue_b32(), acc_22, sb_acc_0_s1, scale_vec_1); + // acc_66 = svmla_s32_m(svptrue_b32(), acc_66, sb_acc_2_s0, scale_vec_0); + // acc_66 = svmla_s32_m(svptrue_b32(), acc_66, sb_acc_2_s1, scale_vec_1); + // } + // if(cp == 3) { + // acc_33 = svmla_s32_m(svptrue_b32(), acc_33, sb_acc_0_s0, scale_vec_0); + // acc_33 = svmla_s32_m(svptrue_b32(), acc_33, sb_acc_0_s1, scale_vec_1); + // acc_77 = svmla_s32_m(svptrue_b32(), acc_77, sb_acc_2_s0, scale_vec_0); + // acc_77 = svmla_s32_m(svptrue_b32(), acc_77, sb_acc_2_s1, scale_vec_1); + // } + // } // for cp + // } // for sb + + // acc_00 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_00, svext_s32(acc_00, acc_00, 4)); + // acc_11 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_11, svext_s32(acc_11, acc_11, 4)); + // acc_22 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_22, svext_s32(acc_22, acc_22, 4)); + // acc_33 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_33, svext_s32(acc_33, acc_33, 4)); + // acc_44 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_44, svext_s32(acc_44, acc_44, 4)); + // acc_55 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_55, svext_s32(acc_55, acc_55, 4)); + // acc_66 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_66, svext_s32(acc_66, acc_66, 4)); + // acc_77 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_77, svext_s32(acc_77, acc_77, 4)); + + // svint32_t reorder_acc_01 = svtbl_s32( svzip1_s32( svtrn1_s32(acc_00, acc_11), svtrn1_s32(acc_22, acc_33)), idx1); + // svint32_t reorder_acc_23 = svtbl_s32( svzip1_s32( svtrn2_s32(acc_00, acc_11), svtrn2_s32(acc_22, acc_33)), idx1); + + // svint32_t reorder_acc_45 = svtbl_s32( svzip1_s32( svtrn1_s32(acc_44, acc_55), svtrn1_s32(acc_66, acc_77)), idx1); + // svint32_t reorder_acc_67 = svtbl_s32( svzip1_s32( svtrn2_s32(acc_44, acc_55), svtrn2_s32(acc_66, acc_77)), idx1); + + // // Broadcast q8 scalar + // svfloat32_t q8_d = svdup_f32(q8_ptr[b].d[0]); + // svfloat32_t q6_d = svcvt_f32_f16_x(svptrue_b32(), svzip1_f16( svld1_f16(svptrue_pat_b16(SV_VL8), (const __fp16 *)q6_ptr[b].d), svdup_f16(0))); + // svfloat32_t scale1 = svmul_f32_x(svptrue_b32(), q6_d, q8_d); + // acc_f32_01 = svmla_f32_m(svptrue_b32(), acc_f32_01, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_01), scale1); + + // q8_d = svdup_f32(q8_ptr[b].d[1]); + // scale1 = svmul_f32_x(svptrue_b32(), q6_d, q8_d); + // acc_f32_23 = svmla_f32_m(svptrue_b32(), acc_f32_23, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_23), scale1); + + // q8_d = svdup_f32(q8_ptr[b].d[2]); + // scale1 = svmul_f32_x(svptrue_b32(), q6_d, q8_d); + // acc_f32_45 = svmla_f32_m(svptrue_b32(), acc_f32_45, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_45), scale1); + + // q8_d = svdup_f32(q8_ptr[b].d[3]); + // scale1 = svmul_f32_x(svptrue_b32(), q6_d, q8_d); + // acc_f32_67 = svmla_f32_m(svptrue_b32(), acc_f32_67, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_67), scale1); + + // } // for b + + // // With the previous reorder, the tile is already in the correct memory layout. + // // Predicate for exactly 4 lanes + // svbool_t pg4 = svptrue_pat_b32(SV_VL4); + // for (int i = 0; i < q8_k_blocklen; i++) { + // int row = y * q8_k_blocklen + i; + // for (int j = 0; j < 2; j++) { + // int col = x * ncols_interleaved + j * 4; + // int offset = row * bs + col; + + // if (i == 0 && j == 0) { + // // acc_f32_0 → lower half of acc_f32_01 + // svst1_f32(pg4, s + offset, acc_f32_01); + // } else if (i == 0 && j == 1) { + // // acc_f32_1 → upper half of acc_f32_01 + // svst1_f32(pg4, s + offset, svext_f32(acc_f32_01, acc_f32_01, 4)); + // } else if (i == 1 && j == 0) { + // // acc_f32_2 + // svst1_f32(pg4, s + offset, acc_f32_23); + // } else if (i == 1 && j == 1) { + // // acc_f32_3 + // svst1_f32(pg4, s + offset, svext_f32(acc_f32_23, acc_f32_23, 4)); + // } else if (i == 2 && j == 0) { + // // acc_f32_4 + // svst1_f32(pg4, s + offset, acc_f32_45); + // } else if (i == 2 && j == 1) { + // // acc_f32_5 + // svst1_f32(pg4, s + offset, svext_f32(acc_f32_45, acc_f32_45, 4)); + // } else if (i == 3 && j == 0) { + // // acc_f32_6 + // svst1_f32(pg4, s + offset, acc_f32_67); + // } else if (i == 3 && j == 1) { + // // acc_f32_7 + // svst1_f32(pg4, s + offset, svext_f32(acc_f32_67, acc_f32_67, 4)); + // } + // } + // } + // } // for x + // } // for y + // return; + // } + case 256:{ + + // std::cout << "SVE 256 Called" << std::endl; + const svbool_t pg8 = svptrue_b8(); + const svbool_t pg16_8 = svptrue_pat_b16(SV_VL8); + const svbool_t pg32 = svptrue_b32(); + + const svuint8_t low_nibble = svdup_n_u8(0x0f); + const svuint8_t high_lo2 = svdup_n_u8(0x03); + const svuint8_t high_hi2 = svdup_n_u8(0x30); + const svint8_t q6_bias = svdup_n_s8(32); + const svint32_t zero_s32 = svdup_n_s32(0); + + for (int y = 0; y < nr / q8_k_blocklen; ++y) { + const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + y * nb; + + for (int x = 0; x < nc / ncols_interleaved; ++x) { + const block_q6_Kx8 * GGML_RESTRICT q6_ptr = (const block_q6_Kx8 *) vx + x * nb; + + //each acc handles all columns for one row + svfloat32_t acc_f32_r0 = svdup_n_f32(0); + svfloat32_t acc_f32_r1 = svdup_n_f32(0); + svfloat32_t acc_f32_r2 = svdup_n_f32(0); + svfloat32_t acc_f32_r3 = svdup_n_f32(0); + + for (int b = 0; b < nb; ++b) { + svint32_t acc_c03_r01 = zero_s32; + svint32_t acc_c03_r23 = zero_s32; + svint32_t acc_c47_r01 = zero_s32; + svint32_t acc_c47_r23 = zero_s32; + + for (int half = 0; half < 2; ++half) { + const uint8_t * ql_base = q6_ptr[b].ql + half * 512; + const uint8_t * qh_base = q6_ptr[b].qh + half * 256; + + for (int sb = 0; sb < QK_K / 64; ++sb) { + const int8_t * q8_base_l = q8_ptr[b].qs + half * 512 + sb * 64; + const int8_t * q8_base_h = q8_ptr[b].qs + half * 512 + 256 + sb * 64; + + //load 16 bytes (8B of R0, 8B of R1) and repeat in upper half + //low lanes will multiply with 16 bytes of C0 and high lanes will multiply with 16 bytes of C1 + const svint8_t q8_l_r01_k0 = svld1rq_s8(pg8, q8_base_l + 0); + const svint8_t q8_l_r01_k1 = svld1rq_s8(pg8, q8_base_l + 32); + const svint8_t q8_l_r23_k0 = svld1rq_s8(pg8, q8_base_l + 16); + const svint8_t q8_l_r23_k1 = svld1rq_s8(pg8, q8_base_l + 48); + + const svint8_t q8_h_r01_k0 = svld1rq_s8(pg8, q8_base_h + 0); + const svint8_t q8_h_r01_k1 = svld1rq_s8(pg8, q8_base_h + 32); + const svint8_t q8_h_r23_k0 = svld1rq_s8(pg8, q8_base_h + 16); + const svint8_t q8_h_r23_k1 = svld1rq_s8(pg8, q8_base_h + 48); + + const int ql_off = sb * QK_K / 2; + const int qh_off = ql_off & 255; + + const int scale_idx_l = half * 8 + sb; + const int scale_idx_h = scale_idx_l + 4; + + const svint32_t scales_l_all = svld1sb_s32(pg32, q6_ptr[b].scales + scale_idx_l * 8); + const svint32_t scales_h_all = svld1sb_s32(pg32, q6_ptr[b].scales + scale_idx_h * 8); + + const svint32_t scales_l_c03 = svzip1_s32(scales_l_all, scales_l_all); + const svint32_t scales_h_c03 = svzip1_s32(scales_h_all, scales_h_all); + + { + svuint8_t q6_ql0_01 = svld1_u8(pg8, ql_base + ql_off); + svuint8_t q6_ql1_01 = svld1_u8(pg8, ql_base + ql_off + 64); + svuint8_t q6_qh0_01 = svld1_u8(pg8, qh_base + qh_off); + svuint8_t q6_qh1_01 = svld1_u8(pg8, qh_base + qh_off + 64); + + if (sb > 1) { + q6_qh0_01 = svlsr_n_u8_x(pg8, q6_qh0_01, 2); + q6_qh1_01 = svlsr_n_u8_x(pg8, q6_qh1_01, 2); + } + + const svuint8_t q6_qh0_01_hi = svand_u8_x(pg8, q6_qh0_01, high_hi2); + const svuint8_t q6_qh1_01_hi = svand_u8_x(pg8, q6_qh1_01, high_hi2); + + const svuint8_t q6_l0_01_bits = svorr_u8_x(pg8, svand_u8_x(pg8, q6_ql0_01, low_nibble), svlsl_n_u8_x(pg8, svand_u8_x(pg8, q6_qh0_01, high_lo2), 4)); + const svuint8_t q6_l1_01_bits = svorr_u8_x(pg8, svand_u8_x(pg8, q6_ql1_01, low_nibble), svlsl_n_u8_x(pg8, svand_u8_x(pg8, q6_qh1_01, high_lo2), 4)); + const svuint8_t q6_h0_01_bits = svorr_u8_x(pg8, svlsr_n_u8_x(pg8, q6_ql0_01, 4), q6_qh0_01_hi); + const svuint8_t q6_h1_01_bits = svorr_u8_x(pg8, svlsr_n_u8_x(pg8, q6_ql1_01, 4), q6_qh1_01_hi); + + const svint8_t q6_l0_01 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_l0_01_bits), q6_bias); + const svint8_t q6_l1_01 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_l1_01_bits), q6_bias); + const svint8_t q6_h0_01 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_h0_01_bits), q6_bias); + const svint8_t q6_h1_01 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_h1_01_bits), q6_bias); + + svint32_t dot_l_r01 = svmmla_s32(zero_s32, q6_l0_01, q8_l_r01_k0); + dot_l_r01 = svmmla_s32(dot_l_r01, q6_l1_01, q8_l_r01_k1); + + svint32_t dot_h_r01 = svmmla_s32(zero_s32, q6_h0_01, q8_h_r01_k0); + dot_h_r01 = svmmla_s32(dot_h_r01, q6_h1_01, q8_h_r01_k1); + + svint32_t dot_l_r23 = svmmla_s32(zero_s32, q6_l0_01, q8_l_r23_k0); + dot_l_r23 = svmmla_s32(dot_l_r23, q6_l1_01, q8_l_r23_k1); + + svint32_t dot_h_r23 = svmmla_s32(zero_s32, q6_h0_01, q8_h_r23_k0); + dot_h_r23 = svmmla_s32(dot_h_r23, q6_h1_01, q8_h_r23_k1); + + acc_c03_r01 = svmla_s32_x(pg32, acc_c03_r01, dot_l_r01, scales_l_c03); + acc_c03_r01 = svmla_s32_x(pg32, acc_c03_r01, dot_h_r01, scales_h_c03); + acc_c03_r23 = svmla_s32_x(pg32, acc_c03_r23, dot_l_r23, scales_l_c03); + acc_c03_r23 = svmla_s32_x(pg32, acc_c03_r23, dot_h_r23, scales_h_c03); + } + + const svint32_t scales_l_c47 = svzip2_s32(scales_l_all, scales_l_all); + const svint32_t scales_h_c47 = svzip2_s32(scales_h_all, scales_h_all); + + { + svuint8_t q6_ql0_23 = svld1_u8(pg8, ql_base + ql_off + 32); + svuint8_t q6_ql1_23 = svld1_u8(pg8, ql_base + ql_off + 96); + svuint8_t q6_qh0_23 = svld1_u8(pg8, qh_base + qh_off + 32); + svuint8_t q6_qh1_23 = svld1_u8(pg8, qh_base + qh_off + 96); + + if (sb > 1) { + q6_qh0_23 = svlsr_n_u8_x(pg8, q6_qh0_23, 2); + q6_qh1_23 = svlsr_n_u8_x(pg8, q6_qh1_23, 2); + } + + const svuint8_t q6_qh0_23_hi = svand_u8_x(pg8, q6_qh0_23, high_hi2); + const svuint8_t q6_qh1_23_hi = svand_u8_x(pg8, q6_qh1_23, high_hi2); + + const svuint8_t q6_l0_23_bits = svorr_u8_x(pg8, svand_u8_x(pg8, q6_ql0_23, low_nibble), svlsl_n_u8_x(pg8, svand_u8_x(pg8, q6_qh0_23, high_lo2), 4)); + const svuint8_t q6_l1_23_bits = svorr_u8_x(pg8, svand_u8_x(pg8, q6_ql1_23, low_nibble), svlsl_n_u8_x(pg8, svand_u8_x(pg8, q6_qh1_23, high_lo2), 4)); + const svuint8_t q6_h0_23_bits = svorr_u8_x(pg8, svlsr_n_u8_x(pg8, q6_ql0_23, 4), q6_qh0_23_hi); + const svuint8_t q6_h1_23_bits = svorr_u8_x(pg8, svlsr_n_u8_x(pg8, q6_ql1_23, 4), q6_qh1_23_hi); + + const svint8_t q6_l0_23 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_l0_23_bits), q6_bias); + const svint8_t q6_l1_23 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_l1_23_bits), q6_bias); + const svint8_t q6_h0_23 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_h0_23_bits), q6_bias); + const svint8_t q6_h1_23 = svsub_s8_x(pg8, svreinterpret_s8_u8(q6_h1_23_bits), q6_bias); + + svint32_t dot_l_r01 = svmmla_s32(zero_s32, q6_l0_23, q8_l_r01_k0); + dot_l_r01 = svmmla_s32(dot_l_r01, q6_l1_23, q8_l_r01_k1); + + svint32_t dot_h_r01 = svmmla_s32(zero_s32, q6_h0_23, q8_h_r01_k0); + dot_h_r01 = svmmla_s32(dot_h_r01, q6_h1_23, q8_h_r01_k1); + + svint32_t dot_l_r23 = svmmla_s32(zero_s32, q6_l0_23, q8_l_r23_k0); + dot_l_r23 = svmmla_s32(dot_l_r23, q6_l1_23, q8_l_r23_k1); + + svint32_t dot_h_r23 = svmmla_s32(zero_s32, q6_h0_23, q8_h_r23_k0); + dot_h_r23 = svmmla_s32(dot_h_r23, q6_h1_23, q8_h_r23_k1); + + acc_c47_r01 = svmla_s32_x(pg32, acc_c47_r01, dot_l_r01, scales_l_c47); + acc_c47_r01 = svmla_s32_x(pg32, acc_c47_r01, dot_h_r01, scales_h_c47); + acc_c47_r23 = svmla_s32_x(pg32, acc_c47_r23, dot_l_r23, scales_l_c47); + acc_c47_r23 = svmla_s32_x(pg32, acc_c47_r23, dot_h_r23, scales_h_c47); + } + } + } + + const svint32_t dot_r0 = svuzp1_s32(acc_c03_r01, acc_c47_r01); + const svint32_t dot_r1 = svuzp2_s32(acc_c03_r01, acc_c47_r01); + const svint32_t dot_r2 = svuzp1_s32(acc_c03_r23, acc_c47_r23); + const svint32_t dot_r3 = svuzp2_s32(acc_c03_r23, acc_c47_r23); + + const svuint32_t q6_d_bits = svld1uh_u32(pg32, (const uint16_t *) q6_ptr[b].d); + const svfloat32_t q6_d_f32 = svcvt_f32_f16_x(pg32, svreinterpret_f16_u32(q6_d_bits)); + + const svfloat32_t scale_r0 = svmul_n_f32_x(pg32, q6_d_f32, q8_ptr[b].d[0]); + const svfloat32_t scale_r1 = svmul_n_f32_x(pg32, q6_d_f32, q8_ptr[b].d[1]); + const svfloat32_t scale_r2 = svmul_n_f32_x(pg32, q6_d_f32, q8_ptr[b].d[2]); + const svfloat32_t scale_r3 = svmul_n_f32_x(pg32, q6_d_f32, q8_ptr[b].d[3]); + + acc_f32_r0 = svmla_f32_x(pg32, acc_f32_r0, svcvt_f32_s32_x(pg32, dot_r0), scale_r0); + acc_f32_r1 = svmla_f32_x(pg32, acc_f32_r1, svcvt_f32_s32_x(pg32, dot_r1), scale_r1); + acc_f32_r2 = svmla_f32_x(pg32, acc_f32_r2, svcvt_f32_s32_x(pg32, dot_r2), scale_r2); + acc_f32_r3 = svmla_f32_x(pg32, acc_f32_r3, svcvt_f32_s32_x(pg32, dot_r3), scale_r3); + } + + const int row = y * q8_k_blocklen; + const int col = x * ncols_interleaved; + + svst1_f32(pg32, s + (row + 0) * bs + col, acc_f32_r0); + svst1_f32(pg32, s + (row + 1) * bs + col, acc_f32_r1); + svst1_f32(pg32, s + (row + 2) * bs + col, acc_f32_r2); + svst1_f32(pg32, s + (row + 3) * bs + col, acc_f32_r3); + } + } + return; + } + } +#endif // SVE compile-time end #if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) - constexpr int q8_k_blocklen = 4; const uint8x16_t m4b = vdupq_n_u8(0x0f); const uint8x16_t mask_lo = vdupq_n_u8(0x03); const uint8x16_t mask_hi = vdupq_n_u8(0x30); @@ -4760,7 +5226,7 @@ void ggml_gemm_q6_K_8x8_q8_K(int n, } for (int b = 0; b < nb; b++) { - int32x4_t acc[8]; // rows 01 stored in [0][1][2][3], rows 23 stored in [4][5][6][7] + int32x4_t acc[8]; // rows 01 stored inc [0][1][2][3], rows 23 stored in [4][5][6][7] for (int i = 0; i < 8; i++) { acc[i] = vdupq_n_s32(0); } From 4f9d66f22da2de03f08c9bfc3407b0bbeb4f0958 Mon Sep 17 00:00:00 2001 From: Anubhav Srivastava Date: Tue, 28 Jul 2026 11:57:38 +0530 Subject: [PATCH 2/4] cleanup --- ggml/src/ggml-cpu/arch/arm/repack.cpp | 284 +------------------------- 1 file changed, 4 insertions(+), 280 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cpp index 854bc67d5b9e..976547f791b4 100644 --- a/ggml/src/ggml-cpu/arch/arm/repack.cpp +++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp @@ -4726,7 +4726,6 @@ void ggml_gemm_q6_K_8x8_q8_K(int n, const void * GGML_RESTRICT vy, int nr, int nc) { - // std::cout<<"ggml_gemm_q6_K_8x8_q8_K called"; constexpr int qk = QK_K; const int nb = n / qk; @@ -4744,286 +4743,7 @@ void ggml_gemm_q6_K_8x8_q8_K(int n, #if defined(__aarch64__) && defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8) switch(svcntb() * 8){ - // case 256:{ - // // std::cout<<"SVE 256 called"; - // constexpr int col_pairs = ncols_interleaved / 2; - // const svuint8_t m4b = svdup_n_u8(0x0f); - // const svuint8_t mask_lo = svdup_n_u8(0x03); - // const svuint8_t mask_hi = svdup_n_u8(0x30); - // svint32_t zeros = svdup_s32(0); - // svfloat32_t zeros_fp32 = svdup_f32(0.0f); - // svfloat16_t fp16_zero = svdup_n_f16((__fp16)0.0); - // svbool_t pg32_2 = svwhilelt_b32(0, 2); - // svbool_t pg32_4 = svwhilelt_b32(0, 4); - // svbool_t pg32_8 = svptrue_b32(); - // svbool_t pg16_4 = svwhilelt_b16(0, 4); - // svbool_t pg16_8 = svwhilelt_b16(0, 8); - // svbool_t pg8_32 = svptrue_b8(); - // svbool_t ph = svptrue_pat_b8(SV_VL16); - // svbool_t pl = svnot_b_z(svptrue_b8(), ph); - - // // 8 accumulators: 2 row pairs × 4 col pairs - // svfloat32_t acc_f32_01, acc_f32_23, acc_f32_45, acc_f32_67; - // uint32_t idx_arr[8] = { 0, 2, 4, 6, 1, 3, 5, 7 }; - // svbool_t pg = svptrue_pat_b32(SV_VL8); - // svuint32_t idx = svld1(pg, idx_arr); - - // static const uint32_t idx_data[8] = {0, 4, 2, 6, 1, 5, 3, 7}; - // svuint32_t idx1 = svld1_u32(svptrue_b32(), idx_data); - - // for (int y = 0; y < nr / q8_k_blocklen; y++) { - // const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + (y * nb); - - // for (int x = 0; x < nc / ncols_interleaved; x++) { - // const block_q6_Kx8 * GGML_RESTRICT q6_ptr = (const block_q6_Kx8 *) vx + (x * nb); - - // acc_f32_01 = svdup_n_f32(0); - // acc_f32_23 = svdup_n_f32(0); - // acc_f32_45 = svdup_n_f32(0); - // acc_f32_67 = svdup_n_f32(0); - - // for (int b = 0; b < nb; b++) { - // svint32_t sb_acc_0 = svdup_n_s32(0); - // svint32_t sb_acc_2 = svdup_n_s32(0); - - // svint32_t acc_00 = svdup_n_s32(0); - // svint32_t acc_11 = svdup_n_s32(0); - // svint32_t acc_22 = svdup_n_s32(0); - // svint32_t acc_33 = svdup_n_s32(0); - // svint32_t acc_44 = svdup_n_s32(0); - // svint32_t acc_55 = svdup_n_s32(0); - // svint32_t acc_66 = svdup_n_s32(0); - // svint32_t acc_77 = svdup_n_s32(0); - - // for (int sb = 0; sb < QK_K / 64; sb++) { - // const int8_t * q8_base_1 = q8_ptr[b].qs + sb * 256; - - // svint8_t q8_qs_0 = svadd_s8_x(svptrue_b8(), svld1_s8(ph, q8_base_1 + 0), svld1_s8(pl, q8_base_1 + 112)); - // svint8_t q8_qs_2 = svadd_s8_x(svptrue_b8(), svld1_s8(ph, q8_base_1 + 64), svld1_s8(pl, q8_base_1 + 144)); - // svint8_t q8_qs_4 = svadd_s8_x(svptrue_b8(), svld1_s8(ph, q8_base_1 + 128), svld1_s8(pl, q8_base_1 + 176)); - // svint8_t q8_qs_6 = svadd_s8_x(svptrue_b8(), svld1_s8(ph, q8_base_1 + 192), svld1_s8(pl, q8_base_1 + 208)); - - // svint8_t q8_qs_1 = svadd_s8_x(svptrue_b8(), svld1_s8(ph, q8_base_1 + 16), svld1_s8(pl, q8_base_1 + 128)); - // svint8_t q8_qs_3 = svadd_s8_x(svptrue_b8(), svld1_s8(ph, q8_base_1 + 80), svld1_s8(pl, q8_base_1 + 160)); - // svint8_t q8_qs_5 = svadd_s8_x(svptrue_b8(), svld1_s8(ph, q8_base_1 + 144), svld1_s8(pl, q8_base_1 + 192)); - // svint8_t q8_qs_7 = svadd_s8_x(svptrue_b8(), svld1_s8(ph, q8_base_1 + 208), svld1_s8(pl, q8_base_1 + 224)); - - // const int8_t *base_scale_ptr = q6_ptr[b].scales+(sb/2)*64+(sb%2)*16; - - // // Q4s columns iterated in pairs (01, 23, 45, 67) - // for (int cp = 0; cp < ncols_interleaved / 2; cp++) { - // svint32_t sb_acc_0_s0 = svdup_n_s32(0); - // svint32_t sb_acc_0_s1 = svdup_n_s32(0); - // svint32_t sb_acc_2_s0 = svdup_n_s32(0); - // svint32_t sb_acc_2_s1 = svdup_n_s32(0); - - // svuint8_t q6_qs_cp_00_l = svld1rq_u8(svptrue_b8(), q6_ptr[b].ql + sb * QK_K + 16 * cp + 0); - // svuint8_t q6_qs_cp_01_l = svld1rq_u8(svptrue_b8(), q6_ptr[b].ql + sb * QK_K + 16 * cp + 64); - // svuint8_t q6_qs_cp_02_l = svld1rq_u8(svptrue_b8(), q6_ptr[b].ql + sb * QK_K + 16 * cp + 128); - // svuint8_t q6_qs_cp_03_l = svld1rq_u8(svptrue_b8(), q6_ptr[b].ql + sb * QK_K + 16 * cp + 192); - - // // we divide by 2 because for sb=0, range is [0 to 255] - // // but for sb=2, range is [256, 511] - - // svuint8_t q6_qs_cp_00_h = svld1rq_u8(svptrue_b8(), q6_ptr[b].qh + (sb * QK_K)/2 + 16 * cp + 0); - // svuint8_t q6_qs_cp_01_h = svld1rq_u8(svptrue_b8(), q6_ptr[b].qh + (sb * QK_K)/2 + 16 * cp + 64); - // svuint8_t q6_qs_cp_02_h = svld1rq_u8(svptrue_b8(), q6_ptr[b].qh + (sb * QK_K)/2 + 16 * cp + 128); - // svuint8_t q6_qs_cp_03_h = svld1rq_u8(svptrue_b8(), q6_ptr[b].qh + (sb * QK_K)/2 + 16 * cp + 192); - - // //shift qh by 2 bits if sb == 1 or 3 since those indexes hold qh values at offset 64 - // if(sb==1||sb==3){ - // q6_qs_cp_00_h = svlsr_n_u8_x(pg8_32,q6_qs_cp_00_h, 2); - // q6_qs_cp_01_h = svlsr_n_u8_x(pg8_32,q6_qs_cp_01_h, 2); - // q6_qs_cp_02_h = svlsr_n_u8_x(pg8_32,q6_qs_cp_02_h, 2); - // q6_qs_cp_03_h = svlsr_n_u8_x(pg8_32,q6_qs_cp_03_h, 2); - // } - - // //Create the entire q values from ql and qh - // // svuint8_t ql = svlsr_n_u8_m(pl, svand_u8_m(ph, q6_qs_cp_00_l, m4b), 4); - // // svuint8_t qh = svand_u8_m(pl, svlsl_n_u8_m(ph, svand_u8_m(ph, q6_qs_cp_00_h, mask_lo), 4), mask_hi); - // // svuint8_t q = svreinterpret_s8_u8(svorr_u8_x(pg8_32, ql, qh)); - - // svint8_t q6_qs_cp_00 = svreinterpret_s8_u8( - // svorr_u8_x(pg8_32, - // svlsr_n_u8_m(pl, svand_u8_m(ph, q6_qs_cp_00_l, m4b), 4), //ql part - // svand_u8_m(pl, svlsl_n_u8_m(ph, svand_u8_m(ph, q6_qs_cp_00_h, mask_lo), 4), mask_hi) //qh part - // )); - - // svint8_t q6_qs_cp_01 = svreinterpret_s8_u8( - // svorr_u8_x(pg8_32, - // svlsr_n_u8_m(pl, svand_u8_m(ph, q6_qs_cp_01_l, m4b), 4), //ql part - // svand_u8_m(pl, svlsl_n_u8_m(ph, svand_u8_m(ph, q6_qs_cp_01_h, mask_lo), 4), mask_hi) //qh part - // )); - - // svint8_t q6_qs_cp_02 = svreinterpret_s8_u8( - // svorr_u8_x(pg8_32, - // svlsr_n_u8_m(pl, svand_u8_m(ph, q6_qs_cp_02_l, m4b), 4), //ql part - // svand_u8_m(pl, svlsl_n_u8_m(ph, svand_u8_m(ph, q6_qs_cp_02_h, mask_lo), 4), mask_hi) //qh part - // )); - - // svint8_t q6_qs_cp_03 = svreinterpret_s8_u8( - // svorr_u8_x(pg8_32, - // svlsr_n_u8_m(pl, svand_u8_m(ph, q6_qs_cp_03_l, m4b), 4), //ql part - // svand_u8_m(pl, svlsl_n_u8_m(ph, svand_u8_m(ph, q6_qs_cp_03_h, mask_lo), 4), mask_hi) //qh part - // )); - - // //Signed 32 correction - // q6_qs_cp_00 = svsub_n_s8_x(pg8_32, q6_qs_cp_00, 32); - // q6_qs_cp_01 = svsub_n_s8_x(pg8_32, q6_qs_cp_01, 32); - // q6_qs_cp_02 = svsub_n_s8_x(pg8_32, q6_qs_cp_02, 32); - // q6_qs_cp_03 = svsub_n_s8_x(pg8_32, q6_qs_cp_03, 32); - - // sb_acc_0_s0 = svmmla_s32(sb_acc_0_s0, q6_qs_cp_00, q8_qs_0); - // sb_acc_0_s0 = svmmla_s32(sb_acc_0_s0, q6_qs_cp_01, q8_qs_2); - - // sb_acc_0_s1 = svmmla_s32(sb_acc_0_s1, q6_qs_cp_02, q8_qs_4); - // sb_acc_0_s1 = svmmla_s32(sb_acc_0_s1, q6_qs_cp_03, q8_qs_6); - - // sb_acc_2_s0 = svmmla_s32(sb_acc_2_s0, q6_qs_cp_00, q8_qs_1); - // sb_acc_2_s0 = svmmla_s32(sb_acc_2_s0, q6_qs_cp_01, q8_qs_3); - - // sb_acc_2_s1 = svmmla_s32(sb_acc_2_s1, q6_qs_cp_02, q8_qs_5); - // sb_acc_2_s1 = svmmla_s32(sb_acc_2_s1, q6_qs_cp_03, q8_qs_7); - - // const int scale_col = 2 * cp; - - // int32_t scale_arr_0[8] = { - // //Low lanes - // base_scale_ptr[scale_col + 0], - // base_scale_ptr[scale_col + 0], - // base_scale_ptr[scale_col + 1], - // base_scale_ptr[scale_col + 1], - // //High Lanes - // base_scale_ptr[scale_col + 32], - // base_scale_ptr[scale_col + 32], - // base_scale_ptr[scale_col + 33], - // base_scale_ptr[scale_col + 33], - // }; - - // int32_t scale_arr_1[8] = { - // //Low lanes - // base_scale_ptr[scale_col + 8], - // base_scale_ptr[scale_col + 8], - // base_scale_ptr[scale_col + 9], - // base_scale_ptr[scale_col + 9], - // //High Lanes - // base_scale_ptr[scale_col + 40], - // base_scale_ptr[scale_col + 40], - // base_scale_ptr[scale_col + 41], - // base_scale_ptr[scale_col + 41], - // }; - - // const svint32_t scale_vec_0 = - // svld1_s32(pg32_8, scale_arr_0); - - // const svint32_t scale_vec_1 = - // svld1_s32(pg32_8, scale_arr_1); - - // if(cp == 0) { - // acc_00 = svmla_s32_m(svptrue_b32(), acc_00, sb_acc_0_s0, scale_vec_0); - // acc_00 = svmla_s32_m(svptrue_b32(), acc_00, sb_acc_0_s1, scale_vec_1); - // acc_44 = svmla_s32_m(svptrue_b32(), acc_44, sb_acc_2_s0, scale_vec_0); - // acc_44 = svmla_s32_m(svptrue_b32(), acc_44, sb_acc_2_s1, scale_vec_1); - // } - // if(cp == 1) { - // acc_11 = svmla_s32_m(svptrue_b32(), acc_11, sb_acc_0_s0, scale_vec_0); - // acc_11 = svmla_s32_m(svptrue_b32(), acc_11, sb_acc_0_s1, scale_vec_1); - // acc_55 = svmla_s32_m(svptrue_b32(), acc_55, sb_acc_2_s0, scale_vec_0); - // acc_55 = svmla_s32_m(svptrue_b32(), acc_55, sb_acc_2_s1, scale_vec_1); - // } - // if(cp == 2) { - // acc_22 = svmla_s32_m(svptrue_b32(), acc_22, sb_acc_0_s0, scale_vec_0); - // acc_22 = svmla_s32_m(svptrue_b32(), acc_22, sb_acc_0_s1, scale_vec_1); - // acc_66 = svmla_s32_m(svptrue_b32(), acc_66, sb_acc_2_s0, scale_vec_0); - // acc_66 = svmla_s32_m(svptrue_b32(), acc_66, sb_acc_2_s1, scale_vec_1); - // } - // if(cp == 3) { - // acc_33 = svmla_s32_m(svptrue_b32(), acc_33, sb_acc_0_s0, scale_vec_0); - // acc_33 = svmla_s32_m(svptrue_b32(), acc_33, sb_acc_0_s1, scale_vec_1); - // acc_77 = svmla_s32_m(svptrue_b32(), acc_77, sb_acc_2_s0, scale_vec_0); - // acc_77 = svmla_s32_m(svptrue_b32(), acc_77, sb_acc_2_s1, scale_vec_1); - // } - // } // for cp - // } // for sb - - // acc_00 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_00, svext_s32(acc_00, acc_00, 4)); - // acc_11 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_11, svext_s32(acc_11, acc_11, 4)); - // acc_22 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_22, svext_s32(acc_22, acc_22, 4)); - // acc_33 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_33, svext_s32(acc_33, acc_33, 4)); - // acc_44 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_44, svext_s32(acc_44, acc_44, 4)); - // acc_55 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_55, svext_s32(acc_55, acc_55, 4)); - // acc_66 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_66, svext_s32(acc_66, acc_66, 4)); - // acc_77 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_77, svext_s32(acc_77, acc_77, 4)); - - // svint32_t reorder_acc_01 = svtbl_s32( svzip1_s32( svtrn1_s32(acc_00, acc_11), svtrn1_s32(acc_22, acc_33)), idx1); - // svint32_t reorder_acc_23 = svtbl_s32( svzip1_s32( svtrn2_s32(acc_00, acc_11), svtrn2_s32(acc_22, acc_33)), idx1); - - // svint32_t reorder_acc_45 = svtbl_s32( svzip1_s32( svtrn1_s32(acc_44, acc_55), svtrn1_s32(acc_66, acc_77)), idx1); - // svint32_t reorder_acc_67 = svtbl_s32( svzip1_s32( svtrn2_s32(acc_44, acc_55), svtrn2_s32(acc_66, acc_77)), idx1); - - // // Broadcast q8 scalar - // svfloat32_t q8_d = svdup_f32(q8_ptr[b].d[0]); - // svfloat32_t q6_d = svcvt_f32_f16_x(svptrue_b32(), svzip1_f16( svld1_f16(svptrue_pat_b16(SV_VL8), (const __fp16 *)q6_ptr[b].d), svdup_f16(0))); - // svfloat32_t scale1 = svmul_f32_x(svptrue_b32(), q6_d, q8_d); - // acc_f32_01 = svmla_f32_m(svptrue_b32(), acc_f32_01, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_01), scale1); - - // q8_d = svdup_f32(q8_ptr[b].d[1]); - // scale1 = svmul_f32_x(svptrue_b32(), q6_d, q8_d); - // acc_f32_23 = svmla_f32_m(svptrue_b32(), acc_f32_23, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_23), scale1); - - // q8_d = svdup_f32(q8_ptr[b].d[2]); - // scale1 = svmul_f32_x(svptrue_b32(), q6_d, q8_d); - // acc_f32_45 = svmla_f32_m(svptrue_b32(), acc_f32_45, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_45), scale1); - - // q8_d = svdup_f32(q8_ptr[b].d[3]); - // scale1 = svmul_f32_x(svptrue_b32(), q6_d, q8_d); - // acc_f32_67 = svmla_f32_m(svptrue_b32(), acc_f32_67, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_67), scale1); - - // } // for b - - // // With the previous reorder, the tile is already in the correct memory layout. - // // Predicate for exactly 4 lanes - // svbool_t pg4 = svptrue_pat_b32(SV_VL4); - // for (int i = 0; i < q8_k_blocklen; i++) { - // int row = y * q8_k_blocklen + i; - // for (int j = 0; j < 2; j++) { - // int col = x * ncols_interleaved + j * 4; - // int offset = row * bs + col; - - // if (i == 0 && j == 0) { - // // acc_f32_0 → lower half of acc_f32_01 - // svst1_f32(pg4, s + offset, acc_f32_01); - // } else if (i == 0 && j == 1) { - // // acc_f32_1 → upper half of acc_f32_01 - // svst1_f32(pg4, s + offset, svext_f32(acc_f32_01, acc_f32_01, 4)); - // } else if (i == 1 && j == 0) { - // // acc_f32_2 - // svst1_f32(pg4, s + offset, acc_f32_23); - // } else if (i == 1 && j == 1) { - // // acc_f32_3 - // svst1_f32(pg4, s + offset, svext_f32(acc_f32_23, acc_f32_23, 4)); - // } else if (i == 2 && j == 0) { - // // acc_f32_4 - // svst1_f32(pg4, s + offset, acc_f32_45); - // } else if (i == 2 && j == 1) { - // // acc_f32_5 - // svst1_f32(pg4, s + offset, svext_f32(acc_f32_45, acc_f32_45, 4)); - // } else if (i == 3 && j == 0) { - // // acc_f32_6 - // svst1_f32(pg4, s + offset, acc_f32_67); - // } else if (i == 3 && j == 1) { - // // acc_f32_7 - // svst1_f32(pg4, s + offset, svext_f32(acc_f32_67, acc_f32_67, 4)); - // } - // } - // } - // } // for x - // } // for y - // return; - // } case 256:{ - - // std::cout << "SVE 256 Called" << std::endl; const svbool_t pg8 = svptrue_b8(); const svbool_t pg16_8 = svptrue_pat_b16(SV_VL8); const svbool_t pg32 = svptrue_b32(); @@ -5173,6 +4893,10 @@ void ggml_gemm_q6_K_8x8_q8_K(int n, } } + // acc_c03_r01 -> c0-r0,c0-r1,c1-r0,c1-r1,c2-r0,c2-r1,c3-r0,c3-r1 + // acc_c47_r01 -> c4-r0,c4-r1,c5-r0,c5-r1,c6-r0,c6-r1,c7-r0,c7-r1 + // svuzp1_s32 picks even lanes -> c0-r0,c1-r0,c2-r0,c3-r0,c4-r0,c5-r0,c6-r0,c7-r0 + // similarly svuzp2_s32 picks remaining values from odd lanes const svint32_t dot_r0 = svuzp1_s32(acc_c03_r01, acc_c47_r01); const svint32_t dot_r1 = svuzp2_s32(acc_c03_r01, acc_c47_r01); const svint32_t dot_r2 = svuzp1_s32(acc_c03_r23, acc_c47_r23); From 85b2111ba9f5eabdc6554ec1c8ed0185c4ad5119 Mon Sep 17 00:00:00 2001 From: Anubhav Srivastava Date: Tue, 28 Jul 2026 12:03:50 +0530 Subject: [PATCH 3/4] review comments --- ggml/src/ggml-cpu/arch/arm/repack.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cpp index 976547f791b4..17e512a03e4b 100644 --- a/ggml/src/ggml-cpu/arch/arm/repack.cpp +++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp @@ -14,7 +14,6 @@ #include #include // for qsort #include // for GGML_ASSERT -#include #define GGML_CPU_CLANG_WORKAROUND #include "../../repack.h" @@ -4950,7 +4949,7 @@ void ggml_gemm_q6_K_8x8_q8_K(int n, } for (int b = 0; b < nb; b++) { - int32x4_t acc[8]; // rows 01 stored inc [0][1][2][3], rows 23 stored in [4][5][6][7] + int32x4_t acc[8]; // rows 01 stored in [0][1][2][3], rows 23 stored in [4][5][6][7] for (int i = 0; i < 8; i++) { acc[i] = vdupq_n_s32(0); } From 3c115d7385deebcb000b8530b73c1f3190133a1a Mon Sep 17 00:00:00 2001 From: Anubhav Srivastava Date: Tue, 18 Aug 2026 13:52:38 +0530 Subject: [PATCH 4/4] review comment addressed - removed pg16_8 --- ggml/src/ggml-cpu/arch/arm/repack.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cpp index 17e512a03e4b..462817152c59 100644 --- a/ggml/src/ggml-cpu/arch/arm/repack.cpp +++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp @@ -4744,7 +4744,6 @@ void ggml_gemm_q6_K_8x8_q8_K(int n, switch(svcntb() * 8){ case 256:{ const svbool_t pg8 = svptrue_b8(); - const svbool_t pg16_8 = svptrue_pat_b16(SV_VL8); const svbool_t pg32 = svptrue_b32(); const svuint8_t low_nibble = svdup_n_u8(0x0f);