From 4efdf6c47b399c14dceccfbaed1ea873e196f397 Mon Sep 17 00:00:00 2001 From: Anubhav Srivastava Date: Mon, 18 May 2026 12:52:47 +0530 Subject: [PATCH 01/19] Test Commit - Building and Execution Not verified --- ggml/src/ggml-cpu/arch/arm/repack.cpp | 732 +++++++++++++++++--------- 1 file changed, 479 insertions(+), 253 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cpp index a7534443091f..d7ebd59876cf 100644 --- a/ggml/src/ggml-cpu/arch/arm/repack.cpp +++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp @@ -3771,317 +3771,543 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, UNUSED(blocklen); #if defined(__aarch64__) && defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8) - if (svcntb() * 8 == 256) { - constexpr int q8_k_blocklen = 4; - const svuint8_t m4b_1 = svdup_n_u8(0x0f); - // 8 accumulators: 2 row pairs × 4 col pairs - svfloat32_t acc_f32_01, acc_f32_23, acc_f32_45, acc_f32_67; - uint32_t idx_arr[8] = { 0, 2, 4, 6, 1, 3, 5, 7 }; - svbool_t pg = svptrue_pat_b32(SV_VL8); - svuint32_t idx = svld1(pg, idx_arr); + + constexpr int q8_k_blocklen = 4; + + switch(svcntb() * 8){ + case 256: + std::cout << "VL is 256" << std::endl; + // constexpr int q8_k_blocklen = 4; + const svuint8_t m4b_1 = svdup_n_u8(0x0f); + + //SV_VL8 would enable lower 8 lanes, since we're only loading 8 VALUES here this should work for higher VLs as well + svfloat32_t acc_f32_01, acc_f32_23, acc_f32_45, acc_f32_67; + uint32_t idx_arr[8] = { 0, 2, 4, 6, 1, 3, 5, 7 }; + svbool_t pg = svptrue_pat_b32(SV_VL8); + svuint32_t idx = svld1(pg, idx_arr); + + static const uint32_t idx_data[8] = {0, 4, 2, 6, 1, 5, 3, 7}; + svuint32_t idx1 = svld1_u32(svptrue_b32(), idx_data); + + for (int y = 0; y < nr / q8_k_blocklen; y++) { + const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + (y * nb); + + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_q4_Kx8 * GGML_RESTRICT q4_ptr = (const block_q4_Kx8 *) vx + (x * nb); + + acc_f32_01 = svdup_n_f32(0); + acc_f32_23 = svdup_n_f32(0); + acc_f32_45 = svdup_n_f32(0); + acc_f32_67 = svdup_n_f32(0); + + for (int b = 0; b < nb; b++) { //nb is number of quantization blocks + // bsums pairs belongs to the same q8_k subblock + // 64 elements loaded and made sum of 0-7 and 8-15 sum || 16-23 and 24 - 31 sum + const int16x8_t bsums[4]{ + vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 0), vld1q_s16(q8_ptr[b].bsums + 16 * 0 + 8)), + vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 1), vld1q_s16(q8_ptr[b].bsums + 16 * 1 + 8)), + vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 2), vld1q_s16(q8_ptr[b].bsums + 16 * 2 + 8)), + vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 3), vld1q_s16(q8_ptr[b].bsums + 16 * 3 + 8)), + }; - static const uint32_t idx_data[8] = {0, 4, 2, 6, 1, 5, 3, 7}; - svuint32_t idx1 = svld1_u32(svptrue_b32(), idx_data); + int32_t bsums_arr32[4][8]; - for (int y = 0; y < nr / q8_k_blocklen; y++) { - const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + (y * nb); + for (int q8_row = 0; q8_row < 4; q8_row++) { + int16x8_t v16 = bsums[q8_row]; - for (int x = 0; x < nc / ncols_interleaved; x++) { - const block_q4_Kx8 * GGML_RESTRICT q4_ptr = (const block_q4_Kx8 *) vx + (x * nb); + // low 4 + int32x4_t v32_lo = vmovl_s16(vget_low_s16(v16)); + vst1q_s32(&bsums_arr32[q8_row][0], v32_lo); - acc_f32_01 = svdup_n_f32(0); - acc_f32_23 = svdup_n_f32(0); - acc_f32_45 = svdup_n_f32(0); - acc_f32_67 = svdup_n_f32(0); + // high 4 + int32x4_t v32_hi = vmovl_s16(vget_high_s16(v16)); + vst1q_s32(&bsums_arr32[q8_row][4], v32_hi); + } - for (int b = 0; b < nb; b++) { - // bsums pairs belongs to the same q8_k subblock - // 64 elements loaded and made sum of 0-7 and 8-15 sum || 16-23 and 24 - 31 sum - const int16x8_t bsums[4]{ - vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 0), vld1q_s16(q8_ptr[b].bsums + 16 * 0 + 8)), - vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 1), vld1q_s16(q8_ptr[b].bsums + 16 * 1 + 8)), - vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 2), vld1q_s16(q8_ptr[b].bsums + 16 * 2 + 8)), - vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 3), vld1q_s16(q8_ptr[b].bsums + 16 * 3 + 8)), - }; + svint32_t sb_acc_0 = svdup_n_s32(0); + svint32_t sb_acc_2 = svdup_n_s32(0); + + svint32_t acc_00 = svdup_n_s32(0); + svint32_t acc_11 = svdup_n_s32(0); + svint32_t acc_22 = svdup_n_s32(0); + svint32_t acc_33 = svdup_n_s32(0); + svint32_t acc_44 = svdup_n_s32(0); + svint32_t acc_55 = svdup_n_s32(0); + svint32_t acc_66 = svdup_n_s32(0); + svint32_t acc_77 = svdup_n_s32(0); + + svint32_t bias_acc_00 = svdup_n_s32(0); + svint32_t bias_acc_22 = svdup_n_s32(0); + svint32_t bias_acc_44 = svdup_n_s32(0); + svint32_t bias_acc_66 = svdup_n_s32(0); + + for (int sb = 0; sb < QK_K / 64; sb++) { + // Need scales for the low and high nibbles + // 2 * 12 = 24 bytes per subblock, 4 sbs -> 4 * 24 = 96 bytes total + svint32_t block_scale_0, block_scale_1, block_scale_2, block_scale_3; + svint32_t q4sb_mins_0, q4sb_mins_1; + { + // 2-superblock I am working on + const int offset = sb * 24 + 0 * 12; + const uint8_t * scales_in = &q4_ptr[b].scales[offset]; - int32_t bsums_arr32[4][8]; + const int offset1 = sb * 24 + 12; + const uint8_t * scales_in1 = &q4_ptr[b].scales[offset1]; - for (int q8_row = 0; q8_row < 4; q8_row++) { - int16x8_t v16 = bsums[q8_row]; + constexpr uint32_t kmask1 = 0x3f3f3f3f; + constexpr uint32_t kmask2 = 0x0f0f0f0f; + constexpr uint32_t kmask3 = 0x03030303; + constexpr uint8_t scales_size = 12; - // low 4 - int32x4_t v32_lo = vmovl_s16(vget_low_s16(v16)); - vst1q_s32(&bsums_arr32[q8_row][0], v32_lo); + uint32_t sm[3]; + memcpy(sm, scales_in, scales_size); - // high 4 - int32x4_t v32_hi = vmovl_s16(vget_high_s16(v16)); - vst1q_s32(&bsums_arr32[q8_row][4], v32_hi); - } + uint32_t sm1[3]; + memcpy(sm1, scales_in1, scales_size); - svint32_t sb_acc_0 = svdup_n_s32(0); - svint32_t sb_acc_2 = svdup_n_s32(0); + const uint32_t mins_0_3 = sm[1] & kmask1; + const uint32_t mins_4_7 = ((sm[2] >> 4) & kmask2) | (((sm[1] >> 6) & kmask3) << 4); - svint32_t acc_00 = svdup_n_s32(0); - svint32_t acc_11 = svdup_n_s32(0); - svint32_t acc_22 = svdup_n_s32(0); - svint32_t acc_33 = svdup_n_s32(0); - svint32_t acc_44 = svdup_n_s32(0); - svint32_t acc_55 = svdup_n_s32(0); - svint32_t acc_66 = svdup_n_s32(0); - svint32_t acc_77 = svdup_n_s32(0); + const uint32_t mins_0_3_1 = sm1[1] & kmask1; + const uint32_t mins_4_7_1 = ((sm1[2] >> 4) & kmask2) | (((sm1[1] >> 6) & kmask3) << 4); - svint32_t bias_acc_00 = svdup_n_s32(0); - svint32_t bias_acc_22 = svdup_n_s32(0); - svint32_t bias_acc_44 = svdup_n_s32(0); - svint32_t bias_acc_66 = svdup_n_s32(0); + svuint32_t mins_u32_temp = svzip1_u32(svdup_n_u32(mins_0_3), svdup_n_u32(mins_4_7)); + svuint32_t mins_u32_temp_1 = svzip1_u32(svdup_n_u32(mins_0_3_1), svdup_n_u32(mins_4_7_1)); - for (int sb = 0; sb < QK_K / 64; sb++) { - // Need scales for the low and high nibbles - // 2 * 12 = 24 bytes per subblock, 4 sbs -> 4 * 24 = 96 bytes total - svint32_t block_scale_0, block_scale_1, block_scale_2, block_scale_3; - svint32_t q4sb_mins_0, q4sb_mins_1; - { - // 2-superblock I am working on - const int offset = sb * 24 + 0 * 12; - const uint8_t * scales_in = &q4_ptr[b].scales[offset]; - - const int offset1 = sb * 24 + 12; - const uint8_t * scales_in1 = &q4_ptr[b].scales[offset1]; - - constexpr uint32_t kmask1 = 0x3f3f3f3f; - constexpr uint32_t kmask2 = 0x0f0f0f0f; - constexpr uint32_t kmask3 = 0x03030303; - constexpr uint8_t scales_size = 12; - - uint32_t sm[3]; - memcpy(sm, scales_in, scales_size); - - uint32_t sm1[3]; - memcpy(sm1, scales_in1, scales_size); - - const uint32_t mins_0_3 = sm[1] & kmask1; - const uint32_t mins_4_7 = ((sm[2] >> 4) & kmask2) | (((sm[1] >> 6) & kmask3) << 4); - - const uint32_t mins_0_3_1 = sm1[1] & kmask1; - const uint32_t mins_4_7_1 = ((sm1[2] >> 4) & kmask2) | (((sm1[1] >> 6) & kmask3) << 4); - - svuint32_t mins_u32_temp = svzip1_u32(svdup_n_u32(mins_0_3), svdup_n_u32(mins_4_7)); - svuint32_t mins_u32_temp_1 = svzip1_u32(svdup_n_u32(mins_0_3_1), svdup_n_u32(mins_4_7_1)); - - /* reinterpret u32 → u8 */ - svuint8_t mins_u8 = svreinterpret_u8_u32(mins_u32_temp); - svuint8_t mins_u8_1 = svreinterpret_u8_u32(mins_u32_temp_1); - - /* widen u8 → u16->u32 (lower half only) */ - svuint32_t mins_u16 = svunpklo_u32(svunpklo_u16(mins_u8)); - svuint32_t mins_u16_1 = svunpklo_u32(svunpklo_u16(mins_u8_1)); - - q4sb_mins_0 = svreinterpret_s32_u32(mins_u16); - q4sb_mins_1 = svreinterpret_s32_u32(mins_u16_1); - - uint32_t scales_u32_0 = sm[0] & kmask1; - uint32_t scales_u32_1 = (sm[2] & kmask2) | (((sm[0] >> 6) & kmask3) << 4); - uint32_t scales_u32_2 = sm1[0] & kmask1; - uint32_t scales_u32_3 = (sm1[2] & kmask2) | (((sm1[0] >> 6) & kmask3) << 4); - - svuint32_t S01 = svdup_n_u32(scales_u32_0); - svuint32_t S23 = svdup_n_u32(scales_u32_1); - svuint32_t R01 = svdup_n_u32(scales_u32_2); - svuint32_t R23 = svdup_n_u32(scales_u32_3); - - svint8_t S01_b = svreinterpret_s8_u32(S01); - svint8_t S23_b = svreinterpret_s8_u32(S23); - svint8_t R01_b = svreinterpret_s8_u32(R01); - svint8_t R23_b = svreinterpret_s8_u32(R23); - - svint32_t S01_d = svunpklo_s32(svunpklo_s16(svzip1_s8(S01_b, S01_b))); - svint32_t R01_d = svunpklo_s32(svunpklo_s16(svzip1_s8(R01_b, R01_b))); - svint32_t S23_d = svunpklo_s32(svunpklo_s16(svzip1_s8(S23_b, S23_b))); - svint32_t R23_d = svunpklo_s32(svunpklo_s16(svzip1_s8(R23_b, R23_b))); - - block_scale_0 = svtbl_s32(svzip1_s32(S01_d, R01_d), idx); - block_scale_1 = svtbl_s32(svzip2_s32(S01_d, R01_d), idx); - block_scale_2 = svtbl_s32(svzip1_s32(S23_d, R23_d), idx); - block_scale_3 = svtbl_s32(svzip2_s32(S23_d, R23_d), idx); - } + /* reinterpret u32 → u8 */ + svuint8_t mins_u8 = svreinterpret_u8_u32(mins_u32_temp); + svuint8_t mins_u8_1 = svreinterpret_u8_u32(mins_u32_temp_1); - const int8_t * q8_base_1 = q8_ptr[b].qs + sb * 256; + /* widen u8 → u16->u32 (lower half only) */ + svuint32_t mins_u16 = svunpklo_u32(svunpklo_u16(mins_u8)); + svuint32_t mins_u16_1 = svunpklo_u32(svunpklo_u16(mins_u8_1)); - // Load 32-byte per row pair, 1 subblock each time - // predicate for activating higher lanes for 16 int8 elements - const svbool_t ph16 = svptrue_pat_b8(SV_VL16); - // predicate for activating lower lanes for 16 int8 elements - const svbool_t pl16 = svnot_b_z(svptrue_b8(), ph16); + q4sb_mins_0 = svreinterpret_s32_u32(mins_u16); + q4sb_mins_1 = svreinterpret_s32_u32(mins_u16_1); - svint8_t q8_qs_0 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 0), svld1_s8(pl16, q8_base_1 + 112)); - svint8_t q8_qs_2 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 32), svld1_s8(pl16, q8_base_1 + 144)); - svint8_t q8_qs_4 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 64), svld1_s8(pl16, q8_base_1 + 176)); - svint8_t q8_qs_6 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 96), svld1_s8(pl16, q8_base_1 + 208)); + uint32_t scales_u32_0 = sm[0] & kmask1; + uint32_t scales_u32_1 = (sm[2] & kmask2) | (((sm[0] >> 6) & kmask3) << 4); + uint32_t scales_u32_2 = sm1[0] & kmask1; + uint32_t scales_u32_3 = (sm1[2] & kmask2) | (((sm1[0] >> 6) & kmask3) << 4); - svint8_t q8_qs_1 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 16), svld1_s8(pl16, q8_base_1 + 128)); - svint8_t q8_qs_3 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 48), svld1_s8(pl16, q8_base_1 + 160)); - svint8_t q8_qs_5 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 80), svld1_s8(pl16, q8_base_1 + 192)); - svint8_t q8_qs_7 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 112), svld1_s8(pl16, q8_base_1 + 224)); + svuint32_t S01 = svdup_n_u32(scales_u32_0); + svuint32_t S23 = svdup_n_u32(scales_u32_1); + svuint32_t R01 = svdup_n_u32(scales_u32_2); + svuint32_t R23 = svdup_n_u32(scales_u32_3); - // Q4s columns iterated in pairs (01, 23, 45, 67) - for (int cp = 0; cp < ncols_interleaved / 2; cp++) { + svint8_t S01_b = svreinterpret_s8_u32(S01); + svint8_t S23_b = svreinterpret_s8_u32(S23); + svint8_t R01_b = svreinterpret_s8_u32(R01); + svint8_t R23_b = svreinterpret_s8_u32(R23); - sb_acc_0 = svdup_n_s32(0); - sb_acc_2 = svdup_n_s32(0); + svint32_t S01_d = svunpklo_s32(svunpklo_s16(svzip1_s8(S01_b, S01_b))); + svint32_t R01_d = svunpklo_s32(svunpklo_s16(svzip1_s8(R01_b, R01_b))); + svint32_t S23_d = svunpklo_s32(svunpklo_s16(svzip1_s8(S23_b, S23_b))); + svint32_t R23_d = svunpklo_s32(svunpklo_s16(svzip1_s8(R23_b, R23_b))); - svuint8_t q4_qs_cp_00 = svld1rq_u8(svptrue_b8(), q4_ptr[b].qs + sb * QK_K + 16 * cp + 0); - svuint8_t q4_qs_cp_01 = svld1rq_u8(svptrue_b8(), q4_ptr[b].qs + sb * QK_K + 16 * cp + 64); - svuint8_t q4_qs_cp_02 = svld1rq_u8(svptrue_b8(), q4_ptr[b].qs + sb * QK_K + 16 * cp + 128); - svuint8_t q4_qs_cp_03 = svld1rq_u8(svptrue_b8(), q4_ptr[b].qs + sb * QK_K + 16 * cp + 192); + block_scale_0 = svtbl_s32(svzip1_s32(S01_d, R01_d), idx); + block_scale_1 = svtbl_s32(svzip2_s32(S01_d, R01_d), idx); + block_scale_2 = svtbl_s32(svzip1_s32(S23_d, R23_d), idx); + block_scale_3 = svtbl_s32(svzip2_s32(S23_d, R23_d), idx); + } //This Q4_K scale computation should also remain the same - svint8_t q4_nibbles_00 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_u8_m(ph16, q4_qs_cp_00, m4b_1), 4)); - svint8_t q4_nibbles_01 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_u8_m(ph16, q4_qs_cp_01, m4b_1), 4)); - svint8_t q4_nibbles_02 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_u8_m(ph16, q4_qs_cp_02, m4b_1), 4)); - svint8_t q4_nibbles_03 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_u8_m(ph16, q4_qs_cp_03, m4b_1), 4)); + const int8_t * q8_base_1 = q8_ptr[b].qs + sb * 256; - sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_00, q8_qs_0); - sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_01, q8_qs_2); + // Load 32-byte per row pair, 1 subblock each time + // predicate for activating higher lanes for 16 int8 elements + const svbool_t ph16 = svptrue_pat_b8(SV_VL16); + // predicate for activating lower lanes for 16 int8 elements + const svbool_t pl16 = svnot_b_z(svptrue_b8(), ph16); - sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_02, q8_qs_4); - sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_03, q8_qs_6); + svint8_t q8_qs_0 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 0), svld1_s8(pl16, q8_base_1 + 112)); + svint8_t q8_qs_2 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 32), svld1_s8(pl16, q8_base_1 + 144)); + svint8_t q8_qs_4 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 64), svld1_s8(pl16, q8_base_1 + 176)); + svint8_t q8_qs_6 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 96), svld1_s8(pl16, q8_base_1 + 208)); - sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_00, q8_qs_1); - sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_01, q8_qs_3); + svint8_t q8_qs_1 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 16), svld1_s8(pl16, q8_base_1 + 128)); + svint8_t q8_qs_3 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 48), svld1_s8(pl16, q8_base_1 + 160)); + svint8_t q8_qs_5 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 80), svld1_s8(pl16, q8_base_1 + 192)); + svint8_t q8_qs_7 = svadd_s8_x(svptrue_b8(), svld1_s8(ph16, q8_base_1 + 112), svld1_s8(pl16, q8_base_1 + 224)); - sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_02, q8_qs_5); - sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_03, q8_qs_7); + // Q4s columns iterated in pairs (01, 23, 45, 67) + for (int cp = 0; cp < ncols_interleaved / 2; cp++) { - if(cp == 0) { - acc_00 = svmla_s32_m(svptrue_b32(), acc_00, sb_acc_0, block_scale_0); - acc_44 = svmla_s32_m(svptrue_b32(), acc_44, sb_acc_2, block_scale_0); - } - if(cp == 1) { - acc_11 = svmla_s32_m(svptrue_b32(), acc_11, sb_acc_0, block_scale_1); - acc_55 = svmla_s32_m(svptrue_b32(), acc_55, sb_acc_2, block_scale_1); - } - if(cp == 2) { - acc_22 = svmla_s32_m(svptrue_b32(), acc_22, sb_acc_0, block_scale_2); - acc_66 = svmla_s32_m(svptrue_b32(), acc_66, sb_acc_2, block_scale_2); - } - if(cp == 3) { - acc_33 = svmla_s32_m(svptrue_b32(), acc_33, sb_acc_0, block_scale_3); - acc_77 = svmla_s32_m(svptrue_b32(), acc_77, sb_acc_2, block_scale_3); + sb_acc_0 = svdup_n_s32(0); + sb_acc_2 = svdup_n_s32(0); + + svuint8_t q4_qs_cp_00 = svld1rq_u8(svptrue_b8(), q4_ptr[b].qs + sb * QK_K + 16 * cp + 0); + svuint8_t q4_qs_cp_01 = svld1rq_u8(svptrue_b8(), q4_ptr[b].qs + sb * QK_K + 16 * cp + 64); + svuint8_t q4_qs_cp_02 = svld1rq_u8(svptrue_b8(), q4_ptr[b].qs + sb * QK_K + 16 * cp + 128); + svuint8_t q4_qs_cp_03 = svld1rq_u8(svptrue_b8(), q4_ptr[b].qs + sb * QK_K + 16 * cp + 192); + + svint8_t q4_nibbles_00 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_u8_m(ph16, q4_qs_cp_00, m4b_1), 4)); + svint8_t q4_nibbles_01 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_u8_m(ph16, q4_qs_cp_01, m4b_1), 4)); + svint8_t q4_nibbles_02 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_u8_m(ph16, q4_qs_cp_02, m4b_1), 4)); + svint8_t q4_nibbles_03 = svreinterpret_s8_u8(svlsr_n_u8_m(pl16, svand_u8_m(ph16, q4_qs_cp_03, m4b_1), 4)); + + sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_00, q8_qs_0); + sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_01, q8_qs_2); + + sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_02, q8_qs_4); + sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_03, q8_qs_6); + + sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_00, q8_qs_1); + sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_01, q8_qs_3); + + sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_02, q8_qs_5); + sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_03, q8_qs_7); + + if(cp == 0) { + acc_00 = svmla_s32_m(svptrue_b32(), acc_00, sb_acc_0, block_scale_0); + acc_44 = svmla_s32_m(svptrue_b32(), acc_44, sb_acc_2, block_scale_0); + } + if(cp == 1) { + acc_11 = svmla_s32_m(svptrue_b32(), acc_11, sb_acc_0, block_scale_1); + acc_55 = svmla_s32_m(svptrue_b32(), acc_55, sb_acc_2, block_scale_1); + } + if(cp == 2) { + acc_22 = svmla_s32_m(svptrue_b32(), acc_22, sb_acc_0, block_scale_2); + acc_66 = svmla_s32_m(svptrue_b32(), acc_66, sb_acc_2, block_scale_2); + } + if(cp == 3) { + acc_33 = svmla_s32_m(svptrue_b32(), acc_33, sb_acc_0, block_scale_3); + acc_77 = svmla_s32_m(svptrue_b32(), acc_77, sb_acc_2, block_scale_3); + } } - } - bias_acc_00 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_00, svdup_n_s32(bsums_arr32[sb][0]), q4sb_mins_0); - bias_acc_00 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_00, svdup_n_s32(bsums_arr32[sb][1]), q4sb_mins_1); + bias_acc_00 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_00, svdup_n_s32(bsums_arr32[sb][0]), q4sb_mins_0); + bias_acc_00 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_00, svdup_n_s32(bsums_arr32[sb][1]), q4sb_mins_1); - bias_acc_22 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_22, svdup_n_s32(bsums_arr32[sb][2]), q4sb_mins_0); - bias_acc_22 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_22, svdup_n_s32(bsums_arr32[sb][3]), q4sb_mins_1); + bias_acc_22 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_22, svdup_n_s32(bsums_arr32[sb][2]), q4sb_mins_0); + bias_acc_22 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_22, svdup_n_s32(bsums_arr32[sb][3]), q4sb_mins_1); - bias_acc_44 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_44, svdup_n_s32(bsums_arr32[sb][4]), q4sb_mins_0); - bias_acc_44 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_44, svdup_n_s32(bsums_arr32[sb][5]), q4sb_mins_1); + bias_acc_44 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_44, svdup_n_s32(bsums_arr32[sb][4]), q4sb_mins_0); + bias_acc_44 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_44, svdup_n_s32(bsums_arr32[sb][5]), q4sb_mins_1); - bias_acc_66 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_66, svdup_n_s32(bsums_arr32[sb][6]), q4sb_mins_0); - bias_acc_66 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_66, svdup_n_s32(bsums_arr32[sb][7]), q4sb_mins_1); - } // for sb + bias_acc_66 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_66, svdup_n_s32(bsums_arr32[sb][6]), q4sb_mins_0); + bias_acc_66 = svmla_s32_m(svptrue_pat_b32(SV_VL8), bias_acc_66, svdup_n_s32(bsums_arr32[sb][7]), q4sb_mins_1); + } // for sb - acc_00 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_00, svext_s32(acc_00, acc_00, 4)); - acc_11 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_11, svext_s32(acc_11, acc_11, 4)); - acc_22 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_22, svext_s32(acc_22, acc_22, 4)); - acc_33 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_33, svext_s32(acc_33, acc_33, 4)); - acc_44 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_44, svext_s32(acc_44, acc_44, 4)); - acc_55 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_55, svext_s32(acc_55, acc_55, 4)); - acc_66 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_66, svext_s32(acc_66, acc_66, 4)); - acc_77 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_77, svext_s32(acc_77, acc_77, 4)); + acc_00 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_00, svext_s32(acc_00, acc_00, 4)); + acc_11 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_11, svext_s32(acc_11, acc_11, 4)); + acc_22 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_22, svext_s32(acc_22, acc_22, 4)); + acc_33 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_33, svext_s32(acc_33, acc_33, 4)); + acc_44 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_44, svext_s32(acc_44, acc_44, 4)); + acc_55 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_55, svext_s32(acc_55, acc_55, 4)); + acc_66 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_66, svext_s32(acc_66, acc_66, 4)); + acc_77 = svadd_s32_z(svptrue_pat_b32(SV_VL4), acc_77, svext_s32(acc_77, acc_77, 4)); - svint32_t reorder_acc_01 = svtbl_s32( svzip1_s32( svtrn1_s32(acc_00, acc_11), svtrn1_s32(acc_22, acc_33)), idx1); - svint32_t reorder_acc_23 = svtbl_s32( svzip1_s32( svtrn2_s32(acc_00, acc_11), svtrn2_s32(acc_22, acc_33)), idx1); + svint32_t reorder_acc_01 = svtbl_s32( svzip1_s32( svtrn1_s32(acc_00, acc_11), svtrn1_s32(acc_22, acc_33)), idx1); + svint32_t reorder_acc_23 = svtbl_s32( svzip1_s32( svtrn2_s32(acc_00, acc_11), svtrn2_s32(acc_22, acc_33)), idx1); - svint32_t reorder_acc_45 = svtbl_s32( svzip1_s32( svtrn1_s32(acc_44, acc_55), svtrn1_s32(acc_66, acc_77)), idx1); - svint32_t reorder_acc_67 = svtbl_s32( svzip1_s32( svtrn2_s32(acc_44, acc_55), svtrn2_s32(acc_66, acc_77)), idx1); + svint32_t reorder_acc_45 = svtbl_s32( svzip1_s32( svtrn1_s32(acc_44, acc_55), svtrn1_s32(acc_66, acc_77)), idx1); + svint32_t reorder_acc_67 = svtbl_s32( svzip1_s32( svtrn2_s32(acc_44, acc_55), svtrn2_s32(acc_66, acc_77)), idx1); - // Broadcast q8 scalar - svfloat32_t q8_d = svdup_f32(q8_ptr[b].d[0]); + // Broadcast q8 scalar + svfloat32_t q8_d = svdup_f32(q8_ptr[b].d[0]); - svfloat32_t q4_dmin_temp = svcvt_f32_f16_x(svptrue_b32(), svzip1_f16( svld1_f16(svptrue_pat_b16(SV_VL8), (const __fp16 *)q4_ptr[b].dmin), svdup_f16(0))); + svfloat32_t q4_dmin_temp = svcvt_f32_f16_x(svptrue_b32(), svzip1_f16( svld1_f16(svptrue_pat_b16(SV_VL8), (const __fp16 *)q4_ptr[b].dmin), svdup_f16(0))); - svfloat32_t q4_d_temp = svcvt_f32_f16_x(svptrue_b32(), svzip1_f16( svld1_f16(svptrue_pat_b16(SV_VL8), (const __fp16 *)q4_ptr[b].d), svdup_f16(0))); + svfloat32_t q4_d_temp = svcvt_f32_f16_x(svptrue_b32(), svzip1_f16( svld1_f16(svptrue_pat_b16(SV_VL8), (const __fp16 *)q4_ptr[b].d), svdup_f16(0))); - svfloat32_t scale1 = svmul_f32_x(svptrue_b32(), q4_d_temp, q8_d); - svfloat32_t dmins1 = svmul_f32_x(svptrue_b32(), q4_dmin_temp, q8_d); + svfloat32_t scale1 = svmul_f32_x(svptrue_b32(), q4_d_temp, q8_d); + svfloat32_t dmins1 = svmul_f32_x(svptrue_b32(), q4_dmin_temp, q8_d); - acc_f32_01 = svmls_f32_m(svptrue_b32(), acc_f32_01, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), bias_acc_00), dmins1); - acc_f32_01 = svmla_f32_m(svptrue_b32(), acc_f32_01, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_01), scale1); + acc_f32_01 = svmls_f32_m(svptrue_b32(), acc_f32_01, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), bias_acc_00), dmins1); + acc_f32_01 = svmla_f32_m(svptrue_b32(), acc_f32_01, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_01), scale1); - q8_d = svdup_f32(q8_ptr[b].d[1]); + q8_d = svdup_f32(q8_ptr[b].d[1]); - scale1 = svmul_f32_x(svptrue_b32(), q4_d_temp, q8_d); - dmins1 = svmul_f32_x(svptrue_b32(), q4_dmin_temp, q8_d); + scale1 = svmul_f32_x(svptrue_b32(), q4_d_temp, q8_d); + dmins1 = svmul_f32_x(svptrue_b32(), q4_dmin_temp, q8_d); - acc_f32_23 = svmls_f32_m(svptrue_b32(), acc_f32_23, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), bias_acc_22), dmins1); - acc_f32_23 = svmla_f32_m(svptrue_b32(), acc_f32_23, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_23), scale1); + acc_f32_23 = svmls_f32_m(svptrue_b32(), acc_f32_23, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), bias_acc_22), dmins1); + acc_f32_23 = svmla_f32_m(svptrue_b32(), acc_f32_23, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_23), scale1); - q8_d = svdup_f32(q8_ptr[b].d[2]); + q8_d = svdup_f32(q8_ptr[b].d[2]); - scale1 = svmul_f32_x(svptrue_b32(), q4_d_temp, q8_d); - dmins1 = svmul_f32_x(svptrue_b32(), q4_dmin_temp, q8_d); + scale1 = svmul_f32_x(svptrue_b32(), q4_d_temp, q8_d); + dmins1 = svmul_f32_x(svptrue_b32(), q4_dmin_temp, q8_d); - acc_f32_45 = svmls_f32_m(svptrue_b32(), acc_f32_45, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), bias_acc_44), dmins1); - acc_f32_45 = svmla_f32_m(svptrue_b32(), acc_f32_45, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_45), scale1); + acc_f32_45 = svmls_f32_m(svptrue_b32(), acc_f32_45, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), bias_acc_44), dmins1); + acc_f32_45 = svmla_f32_m(svptrue_b32(), acc_f32_45, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_45), scale1); - q8_d = svdup_f32(q8_ptr[b].d[3]); + q8_d = svdup_f32(q8_ptr[b].d[3]); - scale1 = svmul_f32_x(svptrue_b32(), q4_d_temp, q8_d); - dmins1 = svmul_f32_x(svptrue_b32(), q4_dmin_temp, q8_d); + scale1 = svmul_f32_x(svptrue_b32(), q4_d_temp, q8_d); + dmins1 = svmul_f32_x(svptrue_b32(), q4_dmin_temp, q8_d); - acc_f32_67 = svmls_f32_m(svptrue_b32(), acc_f32_67, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), bias_acc_66), dmins1); - acc_f32_67 = svmla_f32_m(svptrue_b32(), acc_f32_67, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_67), scale1); + acc_f32_67 = svmls_f32_m(svptrue_b32(), acc_f32_67, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), bias_acc_66), dmins1); + acc_f32_67 = svmla_f32_m(svptrue_b32(), acc_f32_67, svcvt_f32_s32_m(svdup_n_f32(0), svptrue_b32(), reorder_acc_67), scale1); - } // for b + } // for b - // With the previous reorder, the tile is already in the correct memory layout. - // Predicate for exactly 4 lanes - svbool_t pg4 = svptrue_pat_b32(SV_VL4); - for (int i = 0; i < q8_k_blocklen; i++) { - int row = y * q8_k_blocklen + i; - for (int j = 0; j < 2; j++) { - int col = x * ncols_interleaved + j * 4; - int offset = row * bs + col; - - if (i == 0 && j == 0) { - // acc_f32_0 → lower half of acc_f32_01 - svst1_f32(pg4, s + offset, acc_f32_01); - } else if (i == 0 && j == 1) { - // acc_f32_1 → upper half of acc_f32_01 - svst1_f32(pg4, s + offset, svext_f32(acc_f32_01, acc_f32_01, 4)); - } else if (i == 1 && j == 0) { - // acc_f32_2 - svst1_f32(pg4, s + offset, acc_f32_23); - } else if (i == 1 && j == 1) { - // acc_f32_3 - svst1_f32(pg4, s + offset, svext_f32(acc_f32_23, acc_f32_23, 4)); - } else if (i == 2 && j == 0) { - // acc_f32_4 - svst1_f32(pg4, s + offset, acc_f32_45); - } else if (i == 2 && j == 1) { - // acc_f32_5 - svst1_f32(pg4, s + offset, svext_f32(acc_f32_45, acc_f32_45, 4)); - } else if (i == 3 && j == 0) { - // acc_f32_6 - svst1_f32(pg4, s + offset, acc_f32_67); - } else if (i == 3 && j == 1) { - // acc_f32_7 - svst1_f32(pg4, s + offset, svext_f32(acc_f32_67, acc_f32_67, 4)); + // With the previous reorder, the tile is already in the correct memory layout. + // Predicate for exactly 4 lanes + svbool_t pg4 = svptrue_pat_b32(SV_VL4); + for (int i = 0; i < q8_k_blocklen; i++) { + int row = y * q8_k_blocklen + i; + for (int j = 0; j < 2; j++) { + int col = x * ncols_interleaved + j * 4; + int offset = row * bs + col; + + if (i == 0 && j == 0) { + // acc_f32_0 → lower half of acc_f32_01 + svst1_f32(pg4, s + offset, acc_f32_01); + } else if (i == 0 && j == 1) { + // acc_f32_1 → upper half of acc_f32_01 + svst1_f32(pg4, s + offset, svext_f32(acc_f32_01, acc_f32_01, 4)); + } else if (i == 1 && j == 0) { + // acc_f32_2 + svst1_f32(pg4, s + offset, acc_f32_23); + } else if (i == 1 && j == 1) { + // acc_f32_3 + svst1_f32(pg4, s + offset, svext_f32(acc_f32_23, acc_f32_23, 4)); + } else if (i == 2 && j == 0) { + // acc_f32_4 + svst1_f32(pg4, s + offset, acc_f32_45); + } else if (i == 2 && j == 1) { + // acc_f32_5 + svst1_f32(pg4, s + offset, svext_f32(acc_f32_45, acc_f32_45, 4)); + } else if (i == 3 && j == 0) { + // acc_f32_6 + svst1_f32(pg4, s + offset, acc_f32_67); + } else if (i == 3 && j == 1) { + // acc_f32_7 + svst1_f32(pg4, s + offset, svext_f32(acc_f32_67, acc_f32_67, 4)); + } } } - } - } // for x - } // for y - return; + } // for x + } // for y + break; + + case 128: + std::cout << "VL is 128" << std::endl; + //constexpr int q8_k_blocklen = 4; + const svuint8_t m4b = svdup_n_u8(0x0f); + svbool_t pg_b16_vl8 = svptrue_pat_b16(SV_VL8); + svbool_t pg_b8_vl16 = svptrue_pat_b8(SV_VL16); + + // 8 accumulators: 2 row pairs × 4 col pairs + svfloat32_t acc_f32[blocklen]; + + for (int y = 0; y < nr / q8_k_blocklen; y++) { + const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + (y * nb); + + for (int x = 0; x < nc / ncols_interleaved; x++) { + const block_q4_Kx8 * GGML_RESTRICT q4_ptr = (const block_q4_Kx8 *) vx + (x * nb); + + for (int i = 0; i < blocklen; i++) { + acc_f32[i] = svdup_n_f32(0); + } + + for (int b = 0; b < nb; b++) { + // bsums pairs belongs to the same q8_k subblock + const svint16_t bsums[4]{ + svaddp_s16_x(pg_b16_vl8, svld1_s16(pg_b16_vl8, q8_ptr[b].bsums + 16 * 0), svld1_s16(pg_b16_vl8, q8_ptr[b].bsums + 16 * 0 + 8)), + svaddp_s16_x(pg_b16_vl8, svld1_s16(pg_b16_vl8, q8_ptr[b].bsums + 16 * 1), svld1_s16(pg_b16_vl8, q8_ptr[b].bsums + 16 * 1 + 8)), + svaddp_s16_x(pg_b16_vl8, svld1_s16(pg_b16_vl8, q8_ptr[b].bsums + 16 * 2), svld1_s16(pg_b16_vl8, q8_ptr[b].bsums + 16 * 2 + 8)), + svaddp_s16_x(pg_b16_vl8, svld1_s16(pg_b16_vl8, q8_ptr[b].bsums + 16 * 3), svld1_s16(pg_b16_vl8, q8_ptr[b].bsums + 16 * 3 + 8)), + }; + int16_t bsums_arr[4][8]; + for (int q8_row = 0; q8_row < 4; q8_row++) { + svst1_s16(pg_b16_vl8, bsums_arr[q8_row], bsums[q8_row]); + } + + // svint32_t sb_acc[4]; // Aux accumulators to store subblock (partial) results + // svint32_t acc[8]; // rows 01 stored in [0][1][2][3] rows 23 stored in [4][5][6][7] + // svint32_t bias_acc[8]; // interleaved bias_acc: [0]->r0 0123, [1]->r0 4567, [2]->r1 0123 ... + + // for (int i = 0; i < 8; i++) { + // acc[i] = svdup_n_f32(0); + // bias_acc[i] = svdup_n_f32(0); + // } + + svint32_t sb_acc_0 = svdup_n_f32(0); + svint32_t sb_acc_1 = svdup_n_f32(0); + svint32_t sb_acc_2 = svdup_n_f32(0); + svint32_t sb_acc_3 = svdup_n_f32(0); + + svint32_t acc_00 = svdup_n_f32(0); + svint32_t acc_11 = svdup_n_f32(0); + svint32_t acc_22 = svdup_n_f32(0); + svint32_t acc_33 = svdup_n_f32(0); + svint32_t acc_44 = svdup_n_f32(0); + svint32_t acc_55 = svdup_n_f32(0); + svint32_t acc_66 = svdup_n_f32(0); + svint32_t acc_77 = svdup_n_f32(0); + + svint32_t bias_acc_00 = svdup_n_f32(0); + svint32_t bias_acc_11 = svdup_n_f32(0); + svint32_t bias_acc_22 = svdup_n_f32(0); + svint32_t bias_acc_33 = svdup_n_f32(0); + svint32_t bias_acc_44 = svdup_n_f32(0); + svint32_t bias_acc_55 = svdup_n_f32(0); + svint32_t bias_acc_66 = svdup_n_f32(0); + svint32_t bias_acc_77 = svdup_n_f32(0); + + + + for (int sb = 0; sb < QK_K / 64; sb++) { + // Need scales for the low and high nibbles + // 2 * 12 = 24 bytes per subblock, 4 sbs -> 4 * 24 = 96 bytes total + int8_t q4sb_scales[2][8]; + svint16_t q4sb_mins[2]; // int16 as its needed for bias_acc later + for (int i = 0; i < 2; i++) { + const int offset = sb * 24 + i * 12; + decode_q_Kx8_6bit_scales_sve(&q4_ptr[b].scales[offset], &q4sb_mins[i], q4sb_scales[i]); + } + + // q8_ptr[b].qs has interleaved Q8 rows (01, 23) + const int8_t * q8_base = q8_ptr[b].qs + sb * 256; + + svint8_t q8_qs_01[8]; + svint8_t q8_qs_23[8]; + + // Load 32-byte per row pair, 1 subblock each time + for (int i = 0; i < 8; i++) { + const int offset = i * 32; // 16 for row 01, 16 for row 23 + q8_qs_01[i] = svld1_s8(pg_b8_vl16, q8_base + offset); + q8_qs_23[i] = svld1_s8(pg_b8_vl16, q8_base + offset + 16); + } + + const svint8_t q8s[2][8] = { + { q8_qs_01[0], q8_qs_01[1], q8_qs_01[2], q8_qs_01[3], + q8_qs_01[4], q8_qs_01[5], q8_qs_01[6], q8_qs_01[7] }, + { q8_qs_23[0], q8_qs_23[1], q8_qs_23[2], q8_qs_23[3], + q8_qs_23[4], q8_qs_23[5], q8_qs_23[6], q8_qs_23[7] }, + }; + + // Q4s columns iterated in pairs (01, 23, 45, 67) + for (int cp = 0; cp < ncols_interleaved / 2; cp++) { + + //This is not allowed - change + for (int i = 0; i < 4; i++) { + sb_acc[i] = svdup_n_s32(0); + } + + svuint8_t q4_qs_cp_0 = svld1_u8(pg_b8_vl16, q4_ptr[b].qs + sb * QK_K + 16 * cp + 0); // 0 .. 7 & 32..39 + svuint8_t q4_qs_cp_1 = svld1_u8(pg_b8_vl16, q4_ptr[b].qs + sb * QK_K + 16 * cp + 64); // 8 ..15 & 40..47 + svuint8_t q4_qs_cp_2 = svld1_u8(pg_b8_vl16, q4_ptr[b].qs + sb * QK_K + 16 * cp + 128); // 16..23 & 48..55 + svuint8_t q4_qs_cp_3 = svld1_u8(pg_b8_vl16, q4_ptr[b].qs + sb * QK_K + 16 * cp + 192); // 24..31 & 56..63 + const int8x16_t q4_nibbles[2][4] = { + { + vreinterpretq_s8_u8(svand(q4_qs_cp_0, m4b)), + vreinterpretq_s8_u8(svand(q4_qs_cp_1, m4b)), + vreinterpretq_s8_u8(svand(q4_qs_cp_2, m4b)), + vreinterpretq_s8_u8(svand(q4_qs_cp_3, m4b)), + }, + { + vreinterpretq_s8_u8(vshrq_n_u8(q4_qs_cp_0, 4)), + vreinterpretq_s8_u8(vshrq_n_u8(q4_qs_cp_1, 4)), + vreinterpretq_s8_u8(vshrq_n_u8(q4_qs_cp_2, 4)), + vreinterpretq_s8_u8(vshrq_n_u8(q4_qs_cp_3, 4)), + } + }; + + // Calculates the Qs muladd of every row pair (rp) rows 01 and 23 of q8 + // for each of the internal 32 qs subblock (blk) + for (int rp = 0; rp < 2; rp++) { + for (int blk = 0; blk < 2; blk++) { + const int8x16_t * q8 = &q8s[rp][4 * blk]; + const int8x16_t * q4 = q4_nibbles[blk]; + int32x4_t acc = sb_acc[2 * rp + blk]; + // mul add for each qs in the same subblock + for (int qs_offset = 0; qs_offset < 4; qs_offset++) { + acc = vmmlaq_s32(acc, q4[qs_offset], q8[qs_offset]); + } + sb_acc[2 * rp + blk] = acc; + } + } + + // Scales[i] corresponds to column i + const int scale_offset = cp * 2; + const int32_t scale_00 = q4sb_scales[0][scale_offset]; + const int32_t scale_01 = q4sb_scales[0][scale_offset + 1]; + const int32_t scale_10 = q4sb_scales[1][scale_offset]; + const int32_t scale_11 = q4sb_scales[1][scale_offset + 1]; + const int32x4_t block_scale_0 = vcombine_s32(vdup_n_s32(scale_00), vdup_n_s32(scale_01)); + const int32x4_t block_scale_1 = vcombine_s32(vdup_n_s32(scale_10), vdup_n_s32(scale_11)); + + acc[cp] = vmlaq_s32(acc[cp], sb_acc[0], block_scale_0); + acc[cp + 4] = vmlaq_s32(acc[cp + 4], sb_acc[2], block_scale_0); + acc[cp] = vmlaq_s32(acc[cp], sb_acc[1], block_scale_1); + acc[cp + 4] = vmlaq_s32(acc[cp + 4], sb_acc[3], block_scale_1); + } + + // Multiply Acc bsum + mins + for (int q8_row = 0; q8_row < 4; q8_row++) { + // Each pair of subblocks share the same bsums + // Load scalar bsum → broadcast to a vector (vdupq_n_s16(s)). + int16x4_t bsums_vec_lo = vdup_n_s16(bsums_arr[sb][q8_row * 2]); + int16x4_t bsums_vec_hi = vdup_n_s16(bsums_arr[sb][q8_row * 2 + 1]); + + bias_acc[2 * q8_row] = + vmlal_s16(bias_acc[2 * q8_row], bsums_vec_lo, vget_low_s16(q4sb_mins[0])); + bias_acc[2 * q8_row] = + vmlal_s16(bias_acc[2 * q8_row], bsums_vec_hi, vget_low_s16(q4sb_mins[1])); + bias_acc[2 * q8_row + 1] = + vmlal_s16(bias_acc[2 * q8_row + 1], bsums_vec_lo, vget_high_s16(q4sb_mins[0])); + bias_acc[2 * q8_row + 1] = + vmlal_s16(bias_acc[2 * q8_row + 1], bsums_vec_hi, vget_high_s16(q4sb_mins[1])); + } + } // for sb + + // Reorder of i8mm output with bias and output layout + for (int i = 0; i < 8; i++) { + int32x2x2_t aux = vzip_s32(vget_low_s32(acc[i]), vget_high_s32(acc[i])); + acc[i] = vcombine_s32(aux.val[0], aux.val[1]); + } + int32x4_t reorder_acc[8] = { + vcombine_s32(vget_low_s32(acc[0]), vget_low_s32(acc[1])), + vcombine_s32(vget_low_s32(acc[2]), vget_low_s32(acc[3])), + vcombine_s32(vget_high_s32(acc[0]), vget_high_s32(acc[1])), + vcombine_s32(vget_high_s32(acc[2]), vget_high_s32(acc[3])), + vcombine_s32(vget_low_s32(acc[4]), vget_low_s32(acc[5])), + vcombine_s32(vget_low_s32(acc[6]), vget_low_s32(acc[7])), + vcombine_s32(vget_high_s32(acc[4]), vget_high_s32(acc[5])), + vcombine_s32(vget_high_s32(acc[6]), vget_high_s32(acc[7])), + }; + + for (int i = 0; i < q8_k_blocklen; i++) { + for (int j = 0; j < 2; j++) { + float32x4_t q8_d = vdupq_n_f32(q8_ptr[b].d[i]); + float32x4_t q4_dmin = vcvt_f32_f16(vld1_f16((const __fp16 *) (q4_ptr[b].dmin + j * 4))); + const float32x4_t dmins = vmulq_f32(q4_dmin, q8_d); + + float32x4_t q4_d = vcvt_f32_f16(vld1_f16((const __fp16 *) (q4_ptr[b].d + j * 4))); + const float32x4_t scale = vmulq_f32(q4_d, q8_d); + + acc_f32[2 * i + j] = vmlsq_f32(acc_f32[2 * i + j], vcvtq_f32_s32(bias_acc[2 * i + j]), dmins); + acc_f32[2 * i + j] = + vmlaq_f32(acc_f32[2 * i + j], vcvtq_f32_s32(reorder_acc[2 * i + j]), scale); + } + } + } // for b + + // With the previous reorder, the tile is already in the correct memory layout. + for (int i = 0; i < q8_k_blocklen; i++) { + int row = y * q8_k_blocklen + i; + for (int j = 0; j < 2; j++) { + int col = x * ncols_interleaved + j * 4; + int offset = row * bs + col; + vst1q_f32(s + offset, acc_f32[2 * i + j]); + } + } + } // for x + } // for y + break; + + default: + std::cout << "Invalid VL. VL is neither 128 nor 256" << std::endl; } #endif // SVE compile-time end #if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) - constexpr int q8_k_blocklen = 4; + //constexpr int q8_k_blocklen = 4; const uint8x16_t m4b = vdupq_n_u8(0x0f); // 8 accumulators: 2 row pairs × 4 col pairs From 03ed7a42db3f490b728ac6e231f08cbd3013d7f3 Mon Sep 17 00:00:00 2001 From: Anubhav Srivastava Date: Mon, 18 May 2026 15:40:45 +0530 Subject: [PATCH 02/19] updated the scale-offset compuation part to SVE, build errors still exist due to bsums array --- ggml/src/ggml-cpu/arch/arm/repack.cpp | 134 +++++++++++++++++++------- 1 file changed, 99 insertions(+), 35 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cpp index d7ebd59876cf..503aac1d5a69 100644 --- a/ggml/src/ggml-cpu/arch/arm/repack.cpp +++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp @@ -14,6 +14,7 @@ #include #include // for qsort #include // for GGML_ASSERT +#include #define GGML_CPU_CLANG_WORKAROUND #include "../../repack.h" @@ -3776,6 +3777,7 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, switch(svcntb() * 8){ case 256: + { std::cout << "VL is 256" << std::endl; // constexpr int q8_k_blocklen = 4; const svuint8_t m4b_1 = svdup_n_u8(0x0f); @@ -4083,16 +4085,18 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, } // for x } // for y break; - + } case 128: + { std::cout << "VL is 128" << std::endl; //constexpr int q8_k_blocklen = 4; const svuint8_t m4b = svdup_n_u8(0x0f); svbool_t pg_b16_vl8 = svptrue_pat_b16(SV_VL8); svbool_t pg_b8_vl16 = svptrue_pat_b8(SV_VL16); + svbool_t pg_b32_vl4 = svptrue_pat_b32(SV_VL4); // 8 accumulators: 2 row pairs × 4 col pairs - svfloat32_t acc_f32[blocklen]; + // svfloat32_t acc_f32[blocklen]; for (int y = 0; y < nr / q8_k_blocklen; y++) { const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + (y * nb); @@ -4100,9 +4104,9 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, for (int x = 0; x < nc / ncols_interleaved; x++) { const block_q4_Kx8 * GGML_RESTRICT q4_ptr = (const block_q4_Kx8 *) vx + (x * nb); - for (int i = 0; i < blocklen; i++) { - acc_f32[i] = svdup_n_f32(0); - } + // for (int i = 0; i < blocklen; i++) { + // acc_f32[i] = svdup_n_f32(0); + // } for (int b = 0; b < nb; b++) { // bsums pairs belongs to the same q8_k subblock @@ -4184,27 +4188,27 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, // Q4s columns iterated in pairs (01, 23, 45, 67) for (int cp = 0; cp < ncols_interleaved / 2; cp++) { - //This is not allowed - change - for (int i = 0; i < 4; i++) { - sb_acc[i] = svdup_n_s32(0); - } + // //This is not allowed - change + // for (int i = 0; i < 4; i++) { + // sb_acc[i] = svdup_n_s32(0); + // } svuint8_t q4_qs_cp_0 = svld1_u8(pg_b8_vl16, q4_ptr[b].qs + sb * QK_K + 16 * cp + 0); // 0 .. 7 & 32..39 svuint8_t q4_qs_cp_1 = svld1_u8(pg_b8_vl16, q4_ptr[b].qs + sb * QK_K + 16 * cp + 64); // 8 ..15 & 40..47 svuint8_t q4_qs_cp_2 = svld1_u8(pg_b8_vl16, q4_ptr[b].qs + sb * QK_K + 16 * cp + 128); // 16..23 & 48..55 svuint8_t q4_qs_cp_3 = svld1_u8(pg_b8_vl16, q4_ptr[b].qs + sb * QK_K + 16 * cp + 192); // 24..31 & 56..63 - const int8x16_t q4_nibbles[2][4] = { + const svint8_t q4_nibbles[2][4] = { { - vreinterpretq_s8_u8(svand(q4_qs_cp_0, m4b)), - vreinterpretq_s8_u8(svand(q4_qs_cp_1, m4b)), - vreinterpretq_s8_u8(svand(q4_qs_cp_2, m4b)), - vreinterpretq_s8_u8(svand(q4_qs_cp_3, m4b)), + svreinterpret_s8_u8(svand_u8_x(pg_b8_vl16, q4_qs_cp_0, m4b)), + svreinterpret_s8_u8(svand_u8_x(pg_b8_vl16, q4_qs_cp_1, m4b)), + svreinterpret_s8_u8(svand_u8_x(pg_b8_vl16, q4_qs_cp_2, m4b)), + svreinterpret_s8_u8(svand_u8_x(pg_b8_vl16, q4_qs_cp_3, m4b)), }, { - vreinterpretq_s8_u8(vshrq_n_u8(q4_qs_cp_0, 4)), - vreinterpretq_s8_u8(vshrq_n_u8(q4_qs_cp_1, 4)), - vreinterpretq_s8_u8(vshrq_n_u8(q4_qs_cp_2, 4)), - vreinterpretq_s8_u8(vshrq_n_u8(q4_qs_cp_3, 4)), + svreinterpret_s8_u8(svlsr_n_u8_x(pg_b8_vl16, q4_qs_cp_0, 4)), + svreinterpret_s8_u8(svlsr_n_u8_x(pg_b8_vl16, q4_qs_cp_1, 4)), + svreinterpret_s8_u8(svlsr_n_u8_x(pg_b8_vl16, q4_qs_cp_2, 4)), + svreinterpret_s8_u8(svlsr_n_u8_x(pg_b8_vl16, q4_qs_cp_3, 4)), } }; @@ -4212,14 +4216,38 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, // for each of the internal 32 qs subblock (blk) for (int rp = 0; rp < 2; rp++) { for (int blk = 0; blk < 2; blk++) { - const int8x16_t * q8 = &q8s[rp][4 * blk]; - const int8x16_t * q4 = q4_nibbles[blk]; - int32x4_t acc = sb_acc[2 * rp + blk]; - // mul add for each qs in the same subblock - for (int qs_offset = 0; qs_offset < 4; qs_offset++) { - acc = vmmlaq_s32(acc, q4[qs_offset], q8[qs_offset]); + const svuint8_t * q8 = &q8s[rp][4 * blk]; + const svuint8_t * q4 = q4_nibbles[blk]; + // int32x4_t acc = sb_acc[2 * rp + blk]; + // // mul add for each qs in the same subblock + // for (int qs_offset = 0; qs_offset < 4; qs_offset++) { + // acc = vmmlaq_s32(acc, q4[qs_offset], q8[qs_offset]); + // } + if(rp == 0 && blk == 0){ + sb_acc_0 = svmmla_s32(sb_acc_0, q4[0], q8[0]); + sb_acc_0 = svmmla_s32(sb_acc_0, q4[1], q8[1]); + sb_acc_0 = svmmla_s32(sb_acc_0, q4[2], q8[2]); + sb_acc_0 = svmmla_s32(sb_acc_0, q4[3], q8[3]); } - sb_acc[2 * rp + blk] = acc; + if(rp == 0 && blk == 1){ + sb_acc_1 = svmmla_s32(sb_acc_0, q4[0], q8[0]); + sb_acc_1 = svmmla_s32(sb_acc_0, q4[1], q8[1]); + sb_acc_1 = svmmla_s32(sb_acc_0, q4[2], q8[2]); + sb_acc_1 = svmmla_s32(sb_acc_0, q4[3], q8[3]); + } + if(rp == 1 && blk == 0){ + sb_acc_2 = svmmla_s32(sb_acc_0, q4[0], q8[0]); + sb_acc_2 = svmmla_s32(sb_acc_0, q4[1], q8[1]); + sb_acc_2 = svmmla_s32(sb_acc_0, q4[2], q8[2]); + sb_acc_2 = svmmla_s32(sb_acc_0, q4[3], q8[3]); + } + if(rp == 1 && blk == 1){ + sb_acc_3 = svmmla_s32(sb_acc_0, q4[0], q8[0]); + sb_acc_3 = svmmla_s32(sb_acc_0, q4[1], q8[1]); + sb_acc_3 = svmmla_s32(sb_acc_0, q4[2], q8[2]); + sb_acc_3 = svmmla_s32(sb_acc_0, q4[3], q8[3]); + } + // sb_acc[2 * rp + blk] = acc; } } @@ -4229,21 +4257,57 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, const int32_t scale_01 = q4sb_scales[0][scale_offset + 1]; const int32_t scale_10 = q4sb_scales[1][scale_offset]; const int32_t scale_11 = q4sb_scales[1][scale_offset + 1]; - const int32x4_t block_scale_0 = vcombine_s32(vdup_n_s32(scale_00), vdup_n_s32(scale_01)); - const int32x4_t block_scale_1 = vcombine_s32(vdup_n_s32(scale_10), vdup_n_s32(scale_11)); - - acc[cp] = vmlaq_s32(acc[cp], sb_acc[0], block_scale_0); - acc[cp + 4] = vmlaq_s32(acc[cp + 4], sb_acc[2], block_scale_0); - acc[cp] = vmlaq_s32(acc[cp], sb_acc[1], block_scale_1); - acc[cp + 4] = vmlaq_s32(acc[cp + 4], sb_acc[3], block_scale_1); + // const int32x4_t block_scale_0 = vcombine_s32(svdup_n_s32(scale_00), svdup_n_s32(scale_01)); + // const int32x4_t block_scale_1 = vcombine_s32(svdup_n_s32(scale_10), svdup_n_s32(scale_11)); + + const svint32_t block_scale_0 = svsel_s32( + svptrue_pat_b32(SV_VL2), + svdup_n_s32(scale_00), + svdup_n_s32(scale_01) + ); + + const svint32_t block_scale_1 = svsel_s32( + svptrue_pat_b32(SV_VL2), + svdup_n_s32(scale_10), + svdup_n_s32(scale_11) + ); + + if(cp == 0) + { + acc_00 = svmla_s32_x(svptrue_pat_b32(SV_VL4), acc_00, sb_acc_0, block_scale_0); + acc_44 = svmla_s32_x(svptrue_pat_b32(SV_VL4), acc_44, sb_acc_2, block_scale_0); + acc_00 = svmla_s32_x(svptrue_pat_b32(SV_VL4), acc_00, sb_acc_1, block_scale_1); + acc_44 = svmla_s32_x(svptrue_pat_b32(SV_VL4), acc_44, sb_acc_3, block_scale_1); + } + if(cp == 1) + { + acc_11 = svmla_s32_x(svptrue_pat_b32(SV_VL4), acc_11, sb_acc_0, block_scale_0); + acc_55 = svmla_s32_x(svptrue_pat_b32(SV_VL4), acc_55, sb_acc_2, block_scale_0); + acc_11 = svmla_s32_x(svptrue_pat_b32(SV_VL4), acc_11, sb_acc_1, block_scale_1); + acc_55 = svmla_s32_x(svptrue_pat_b32(SV_VL4), acc_55, sb_acc_3, block_scale_1); + } + if(cp == 2) + { + acc_22 = svmla_s32_x(svptrue_pat_b32(SV_VL4), acc_22, sb_acc_0, block_scale_0); + acc_66 = svmla_s32_x(svptrue_pat_b32(SV_VL4), acc_66, sb_acc_2, block_scale_0); + acc_22 = svmla_s32_x(svptrue_pat_b32(SV_VL4), acc_22, sb_acc_1, block_scale_1); + acc_66 = svmla_s32_x(svptrue_pat_b32(SV_VL4), acc_66, sb_acc_3, block_scale_1); + } + if(cp == 3) + { + acc_33 = svmla_s32_x(svptrue_pat_b32(SV_VL4), acc_33, sb_acc_0, block_scale_0); + acc_77 = svmla_s32_x(svptrue_pat_b32(SV_VL4), acc_77, sb_acc_2, block_scale_0); + acc_33 = svmla_s32_x(svptrue_pat_b32(SV_VL4), acc_33, sb_acc_1, block_scale_1); + acc_77 = svmla_s32_x(svptrue_pat_b32(SV_VL4), acc_77, sb_acc_3, block_scale_1); + } } // Multiply Acc bsum + mins for (int q8_row = 0; q8_row < 4; q8_row++) { // Each pair of subblocks share the same bsums // Load scalar bsum → broadcast to a vector (vdupq_n_s16(s)). - int16x4_t bsums_vec_lo = vdup_n_s16(bsums_arr[sb][q8_row * 2]); - int16x4_t bsums_vec_hi = vdup_n_s16(bsums_arr[sb][q8_row * 2 + 1]); + svint16_t bsums_vec_lo = svdup_n_s16(bsums_arr[sb][q8_row * 2]); + svint16_t bsums_vec_hi = svdup_n_s16(bsums_arr[sb][q8_row * 2 + 1]); bias_acc[2 * q8_row] = vmlal_s16(bias_acc[2 * q8_row], bsums_vec_lo, vget_low_s16(q4sb_mins[0])); @@ -4300,7 +4364,7 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, } // for x } // for y break; - + } default: std::cout << "Invalid VL. VL is neither 128 nor 256" << std::endl; } From edc8e43a97a11723b71905418f0067349ba2a81e Mon Sep 17 00:00:00 2001 From: Anubhav Srivastava Date: Tue, 19 May 2026 09:37:43 +0530 Subject: [PATCH 03/19] Handled compiler errors until decode_q_Kx8_6bit_scales_sve --- ggml/src/ggml-cpu/arch/arm/repack.cpp | 94 ++++++++++++++++++--------- 1 file changed, 64 insertions(+), 30 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cpp index 503aac1d5a69..358f9053dfaa 100644 --- a/ggml/src/ggml-cpu/arch/arm/repack.cpp +++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp @@ -49,6 +49,35 @@ static inline void decode_q_Kx8_6bit_scales(const uint8_t * scales_in, int16x8_t } #endif + +#if defined(__aarch64__) && defined(__ARM_FEATURE_SVE) && (defined(__ARM_FEATURE_MATMUL_INT8) || defined(__ARM_FEATURE_DOTPROD)) +// Helper for decoding scales and mins of Q4_K and Q5_K block formats +static inline void decode_q_Kx8_6bit_scales_sve(const uint8_t * scales_in, svint16_t * out_mins, int8_t * out_scales) { + constexpr uint32_t kmask1 = 0x3f3f3f3f; + constexpr uint32_t kmask2 = 0x0f0f0f0f; + constexpr uint32_t kmask3 = 0x03030303; + constexpr uint8_t scales_size = 12; + + uint32_t sm[3]; + memcpy(sm, scales_in, scales_size); + + const uint32_t mins_0_3 = sm[1] & kmask1; + const uint32_t mins_4_7 = ((sm[2] >> 4) & kmask2) | (((sm[1] >> 6) & kmask3) << 4); + // const uint32x2_t mins_u32 = { mins_0_3, mins_4_7 }; + uint32_t tmp_mins[2] = { mins_0_3, mins_4_7 }; + svbool_t pg2_u32 = svptrue_pat_b32(SV_VL2); // 2 x uint32_t = 64 bits + svuint32_t mins_u32 = svld1_u32(pg2_u32, tmp_mins); + + + *out_mins = svreinterpret_s16_u16(svunpklo_u16(svreinterpret_u8_u32(mins_u32))); + + uint32_t scales_u32[2]; + scales_u32[0] = sm[0] & kmask1; + scales_u32[1] = (sm[2] & kmask2) | (((sm[0] >> 6) & kmask3) << 4); + memcpy(out_scales, scales_u32, 8); +} +#endif + void ggml_quantize_mat_q8_0_4x4(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { assert(QK8_0 == 32); assert(k % QK8_0 == 0); @@ -4110,15 +4139,16 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, for (int b = 0; b < nb; b++) { // bsums pairs belongs to the same q8_k subblock - const svint16_t bsums[4]{ - svaddp_s16_x(pg_b16_vl8, svld1_s16(pg_b16_vl8, q8_ptr[b].bsums + 16 * 0), svld1_s16(pg_b16_vl8, q8_ptr[b].bsums + 16 * 0 + 8)), - svaddp_s16_x(pg_b16_vl8, svld1_s16(pg_b16_vl8, q8_ptr[b].bsums + 16 * 1), svld1_s16(pg_b16_vl8, q8_ptr[b].bsums + 16 * 1 + 8)), - svaddp_s16_x(pg_b16_vl8, svld1_s16(pg_b16_vl8, q8_ptr[b].bsums + 16 * 2), svld1_s16(pg_b16_vl8, q8_ptr[b].bsums + 16 * 2 + 8)), - svaddp_s16_x(pg_b16_vl8, svld1_s16(pg_b16_vl8, q8_ptr[b].bsums + 16 * 3), svld1_s16(pg_b16_vl8, q8_ptr[b].bsums + 16 * 3 + 8)), + const int16x8_t bsums[4]{ + vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 0), vld1q_s16(q8_ptr[b].bsums + 16 * 0 + 8)), + vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 1), vld1q_s16(q8_ptr[b].bsums + 16 * 1 + 8)), + vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 2), vld1q_s16(q8_ptr[b].bsums + 16 * 2 + 8)), + vpaddq_s16(vld1q_s16(q8_ptr[b].bsums + 16 * 3), vld1q_s16(q8_ptr[b].bsums + 16 * 3 + 8)), }; + int16_t bsums_arr[4][8]; for (int q8_row = 0; q8_row < 4; q8_row++) { - svst1_s16(pg_b16_vl8, bsums_arr[q8_row], bsums[q8_row]); + vst1q_s16(bsums_arr[q8_row], bsums[q8_row]); } // svint32_t sb_acc[4]; // Aux accumulators to store subblock (partial) results @@ -4130,28 +4160,28 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, // bias_acc[i] = svdup_n_f32(0); // } - svint32_t sb_acc_0 = svdup_n_f32(0); - svint32_t sb_acc_1 = svdup_n_f32(0); - svint32_t sb_acc_2 = svdup_n_f32(0); - svint32_t sb_acc_3 = svdup_n_f32(0); - - svint32_t acc_00 = svdup_n_f32(0); - svint32_t acc_11 = svdup_n_f32(0); - svint32_t acc_22 = svdup_n_f32(0); - svint32_t acc_33 = svdup_n_f32(0); - svint32_t acc_44 = svdup_n_f32(0); - svint32_t acc_55 = svdup_n_f32(0); - svint32_t acc_66 = svdup_n_f32(0); - svint32_t acc_77 = svdup_n_f32(0); - - svint32_t bias_acc_00 = svdup_n_f32(0); - svint32_t bias_acc_11 = svdup_n_f32(0); - svint32_t bias_acc_22 = svdup_n_f32(0); - svint32_t bias_acc_33 = svdup_n_f32(0); - svint32_t bias_acc_44 = svdup_n_f32(0); - svint32_t bias_acc_55 = svdup_n_f32(0); - svint32_t bias_acc_66 = svdup_n_f32(0); - svint32_t bias_acc_77 = svdup_n_f32(0); + svint32_t sb_acc_0 = svdup_n_s32(0); + svint32_t sb_acc_1 = svdup_n_s32(0); + svint32_t sb_acc_2 = svdup_n_s32(0); + svint32_t sb_acc_3 = svdup_n_s32(0); + + svint32_t acc_00 = svdup_n_s32(0); + svint32_t acc_11 = svdup_n_s32(0); + svint32_t acc_22 = svdup_n_s32(0); + svint32_t acc_33 = svdup_n_s32(0); + svint32_t acc_44 = svdup_n_s32(0); + svint32_t acc_55 = svdup_n_s32(0); + svint32_t acc_66 = svdup_n_s32(0); + svint32_t acc_77 = svdup_n_s32(0); + + svint32_t bias_acc_00 = svdup_n_s32(0); + svint32_t bias_acc_11 = svdup_n_s32(0); + svint32_t bias_acc_22 = svdup_n_s32(0); + svint32_t bias_acc_33 = svdup_n_s32(0); + svint32_t bias_acc_44 = svdup_n_s32(0); + svint32_t bias_acc_55 = svdup_n_s32(0); + svint32_t bias_acc_66 = svdup_n_s32(0); + svint32_t bias_acc_77 = svdup_n_s32(0); @@ -4159,10 +4189,14 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, // Need scales for the low and high nibbles // 2 * 12 = 24 bytes per subblock, 4 sbs -> 4 * 24 = 96 bytes total int8_t q4sb_scales[2][8]; - svint16_t q4sb_mins[2]; // int16 as its needed for bias_acc later + // svint16_t q4sb_mins[2]; // int16 as its needed for bias_acc later + svint16_t q4sb_mins_0, q4sb_mins_1; // int16 as its needed for bias_acc later for (int i = 0; i < 2; i++) { const int offset = sb * 24 + i * 12; - decode_q_Kx8_6bit_scales_sve(&q4_ptr[b].scales[offset], &q4sb_mins[i], q4sb_scales[i]); + if(i==0) + decode_q_Kx8_6bit_scales_sve(&q4_ptr[b].scales[offset], &q4sb_mins_0, q4sb_scales[i]); + else + decode_q_Kx8_6bit_scales_sve(&q4_ptr[b].scales[offset], &q4sb_mins_1, q4sb_scales[i]); } // q8_ptr[b].qs has interleaved Q8 rows (01, 23) From 71fedfd98d1ee545cfacacdc8254007e980a4335 Mon Sep 17 00:00:00 2001 From: Anubhav Srivastava Date: Tue, 19 May 2026 10:29:54 +0530 Subject: [PATCH 04/19] removal of arrays from q8 mul-add pair computation --- ggml/src/ggml-cpu/arch/arm/repack.cpp | 174 ++++++++++++++++---------- 1 file changed, 111 insertions(+), 63 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cpp index 358f9053dfaa..969fdfc409c9 100644 --- a/ggml/src/ggml-cpu/arch/arm/repack.cpp +++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp @@ -4202,21 +4202,36 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, // q8_ptr[b].qs has interleaved Q8 rows (01, 23) const int8_t * q8_base = q8_ptr[b].qs + sb * 256; - svint8_t q8_qs_01[8]; - svint8_t q8_qs_23[8]; - - // Load 32-byte per row pair, 1 subblock each time - for (int i = 0; i < 8; i++) { - const int offset = i * 32; // 16 for row 01, 16 for row 23 - q8_qs_01[i] = svld1_s8(pg_b8_vl16, q8_base + offset); - q8_qs_23[i] = svld1_s8(pg_b8_vl16, q8_base + offset + 16); - } + // svint8_t q8_qs_01[8]; + svint8_t q8_qs_01_0 = svld1_s8(pg_b8_vl16, q8_base + 0*32); + svint8_t q8_qs_01_1 = svld1_s8(pg_b8_vl16, q8_base + 1*32); + svint8_t q8_qs_01_2 = svld1_s8(pg_b8_vl16, q8_base + 2*32); + svint8_t q8_qs_01_3 = svld1_s8(pg_b8_vl16, q8_base + 3*32); + svint8_t q8_qs_01_4 = svld1_s8(pg_b8_vl16, q8_base + 4*32); + svint8_t q8_qs_01_5 = svld1_s8(pg_b8_vl16, q8_base + 5*32); + svint8_t q8_qs_01_6 = svld1_s8(pg_b8_vl16, q8_base + 6*32); + svint8_t q8_qs_01_7 = svld1_s8(pg_b8_vl16, q8_base + 7*32); + + // svint8_t q8_qs_23[8]; + svint8_t q8_qs_23_0 = svld1_s8(pg_b8_vl16, q8_base + 0*32 + 16); + svint8_t q8_qs_23_1 = svld1_s8(pg_b8_vl16, q8_base + 1*32 + 16); + svint8_t q8_qs_23_2 = svld1_s8(pg_b8_vl16, q8_base + 2*32 + 16); + svint8_t q8_qs_23_3 = svld1_s8(pg_b8_vl16, q8_base + 3*32 + 16); + svint8_t q8_qs_23_4 = svld1_s8(pg_b8_vl16, q8_base + 4*32 + 16); + svint8_t q8_qs_23_5 = svld1_s8(pg_b8_vl16, q8_base + 5*32 + 16); + svint8_t q8_qs_23_6 = svld1_s8(pg_b8_vl16, q8_base + 6*32 + 16); + svint8_t q8_qs_23_7 = svld1_s8(pg_b8_vl16, q8_base + 7*32 + 16); + + // // Load 32-byte per row pair, 1 subblock each time + // for (int i = 0; i < 8; i++) { + // const int offset = i * 32; // 16 for row 01, 16 for row 23 + // q8_qs_01[i] = svld1_s8(pg_b8_vl16, q8_base + offset); + // q8_qs_23[i] = svld1_s8(pg_b8_vl16, q8_base + offset + 16); + // } const svint8_t q8s[2][8] = { - { q8_qs_01[0], q8_qs_01[1], q8_qs_01[2], q8_qs_01[3], - q8_qs_01[4], q8_qs_01[5], q8_qs_01[6], q8_qs_01[7] }, - { q8_qs_23[0], q8_qs_23[1], q8_qs_23[2], q8_qs_23[3], - q8_qs_23[4], q8_qs_23[5], q8_qs_23[6], q8_qs_23[7] }, + { q8_qs_01[0], q8_qs_01[1], q8_qs_01[2], q8_qs_01[3], q8_qs_01[4], q8_qs_01[5], q8_qs_01[6], q8_qs_01[7] }, + { q8_qs_23[0], q8_qs_23[1], q8_qs_23[2], q8_qs_23[3], q8_qs_23[4], q8_qs_23[5], q8_qs_23[6], q8_qs_23[7] }, }; // Q4s columns iterated in pairs (01, 23, 45, 67) @@ -4231,59 +4246,92 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, svuint8_t q4_qs_cp_1 = svld1_u8(pg_b8_vl16, q4_ptr[b].qs + sb * QK_K + 16 * cp + 64); // 8 ..15 & 40..47 svuint8_t q4_qs_cp_2 = svld1_u8(pg_b8_vl16, q4_ptr[b].qs + sb * QK_K + 16 * cp + 128); // 16..23 & 48..55 svuint8_t q4_qs_cp_3 = svld1_u8(pg_b8_vl16, q4_ptr[b].qs + sb * QK_K + 16 * cp + 192); // 24..31 & 56..63 - const svint8_t q4_nibbles[2][4] = { - { - svreinterpret_s8_u8(svand_u8_x(pg_b8_vl16, q4_qs_cp_0, m4b)), - svreinterpret_s8_u8(svand_u8_x(pg_b8_vl16, q4_qs_cp_1, m4b)), - svreinterpret_s8_u8(svand_u8_x(pg_b8_vl16, q4_qs_cp_2, m4b)), - svreinterpret_s8_u8(svand_u8_x(pg_b8_vl16, q4_qs_cp_3, m4b)), - }, - { - svreinterpret_s8_u8(svlsr_n_u8_x(pg_b8_vl16, q4_qs_cp_0, 4)), - svreinterpret_s8_u8(svlsr_n_u8_x(pg_b8_vl16, q4_qs_cp_1, 4)), - svreinterpret_s8_u8(svlsr_n_u8_x(pg_b8_vl16, q4_qs_cp_2, 4)), - svreinterpret_s8_u8(svlsr_n_u8_x(pg_b8_vl16, q4_qs_cp_3, 4)), - } - }; + // const svint8_t q4_nibbles[2][4] = { + // { + // svreinterpret_s8_u8(svand_u8_x(pg_b8_vl16, q4_qs_cp_0, m4b)), + // svreinterpret_s8_u8(svand_u8_x(pg_b8_vl16, q4_qs_cp_1, m4b)), + // svreinterpret_s8_u8(svand_u8_x(pg_b8_vl16, q4_qs_cp_2, m4b)), + // svreinterpret_s8_u8(svand_u8_x(pg_b8_vl16, q4_qs_cp_3, m4b)), + // }, + // { + // svreinterpret_s8_u8(svlsr_n_u8_x(pg_b8_vl16, q4_qs_cp_0, 4)), + // svreinterpret_s8_u8(svlsr_n_u8_x(pg_b8_vl16, q4_qs_cp_1, 4)), + // svreinterpret_s8_u8(svlsr_n_u8_x(pg_b8_vl16, q4_qs_cp_2, 4)), + // svreinterpret_s8_u8(svlsr_n_u8_x(pg_b8_vl16, q4_qs_cp_3, 4)), + // } + // }; + svint8_t q4_nibbles_00 = svreinterpret_s8_u8(svand_u8_x(pg_b8_vl16, q4_qs_cp_0, m4b)); + svint8_t q4_nibbles_01 = svreinterpret_s8_u8(svand_u8_x(pg_b8_vl16, q4_qs_cp_1, m4b)); + svint8_t q4_nibbles_02 = svreinterpret_s8_u8(svand_u8_x(pg_b8_vl16, q4_qs_cp_2, m4b)); + svint8_t q4_nibbles_03 = svreinterpret_s8_u8(svand_u8_x(pg_b8_vl16, q4_qs_cp_3, m4b)); + svint8_t q4_nibbles_10 = svreinterpret_s8_u8(svlsr_n_u8_x(pg_b8_vl16, q4_qs_cp_0, 4)); + svint8_t q4_nibbles_11 = svreinterpret_s8_u8(svlsr_n_u8_x(pg_b8_vl16, q4_qs_cp_0, 4)); + svint8_t q4_nibbles_12 = svreinterpret_s8_u8(svlsr_n_u8_x(pg_b8_vl16, q4_qs_cp_0, 4)); + svint8_t q4_nibbles_13 = svreinterpret_s8_u8(svlsr_n_u8_x(pg_b8_vl16, q4_qs_cp_0, 4)); // Calculates the Qs muladd of every row pair (rp) rows 01 and 23 of q8 // for each of the internal 32 qs subblock (blk) - for (int rp = 0; rp < 2; rp++) { - for (int blk = 0; blk < 2; blk++) { - const svuint8_t * q8 = &q8s[rp][4 * blk]; - const svuint8_t * q4 = q4_nibbles[blk]; - // int32x4_t acc = sb_acc[2 * rp + blk]; - // // mul add for each qs in the same subblock - // for (int qs_offset = 0; qs_offset < 4; qs_offset++) { - // acc = vmmlaq_s32(acc, q4[qs_offset], q8[qs_offset]); - // } - if(rp == 0 && blk == 0){ - sb_acc_0 = svmmla_s32(sb_acc_0, q4[0], q8[0]); - sb_acc_0 = svmmla_s32(sb_acc_0, q4[1], q8[1]); - sb_acc_0 = svmmla_s32(sb_acc_0, q4[2], q8[2]); - sb_acc_0 = svmmla_s32(sb_acc_0, q4[3], q8[3]); - } - if(rp == 0 && blk == 1){ - sb_acc_1 = svmmla_s32(sb_acc_0, q4[0], q8[0]); - sb_acc_1 = svmmla_s32(sb_acc_0, q4[1], q8[1]); - sb_acc_1 = svmmla_s32(sb_acc_0, q4[2], q8[2]); - sb_acc_1 = svmmla_s32(sb_acc_0, q4[3], q8[3]); - } - if(rp == 1 && blk == 0){ - sb_acc_2 = svmmla_s32(sb_acc_0, q4[0], q8[0]); - sb_acc_2 = svmmla_s32(sb_acc_0, q4[1], q8[1]); - sb_acc_2 = svmmla_s32(sb_acc_0, q4[2], q8[2]); - sb_acc_2 = svmmla_s32(sb_acc_0, q4[3], q8[3]); - } - if(rp == 1 && blk == 1){ - sb_acc_3 = svmmla_s32(sb_acc_0, q4[0], q8[0]); - sb_acc_3 = svmmla_s32(sb_acc_0, q4[1], q8[1]); - sb_acc_3 = svmmla_s32(sb_acc_0, q4[2], q8[2]); - sb_acc_3 = svmmla_s32(sb_acc_0, q4[3], q8[3]); - } - // sb_acc[2 * rp + blk] = acc; - } - } + // for (int rp = 0; rp < 2; rp++) { + // for (int blk = 0; blk < 2; blk++) { + // const svuint8_t * q8 = &q8s[rp][4 * blk]; + // const svuint8_t * q4 = q4_nibbles[blk]; + // // int32x4_t acc = sb_acc[2 * rp + blk]; + // // // mul add for each qs in the same subblock + // // for (int qs_offset = 0; qs_offset < 4; qs_offset++) { + // // acc = vmmlaq_s32(acc, q4[qs_offset], q8[qs_offset]); + // // } + // if(rp == 0 && blk == 0){ + // sb_acc_0 = svmmla_s32(sb_acc_0, q4[0], q8[0]); + // sb_acc_0 = svmmla_s32(sb_acc_0, q4[1], q8[1]); + // sb_acc_0 = svmmla_s32(sb_acc_0, q4[2], q8[2]); + // sb_acc_0 = svmmla_s32(sb_acc_0, q4[3], q8[3]); + // } + // if(rp == 0 && blk == 1){ + // sb_acc_1 = svmmla_s32(sb_acc_0, q4[0], q8[0]); + // sb_acc_1 = svmmla_s32(sb_acc_0, q4[1], q8[1]); + // sb_acc_1 = svmmla_s32(sb_acc_0, q4[2], q8[2]); + // sb_acc_1 = svmmla_s32(sb_acc_0, q4[3], q8[3]); + // } + // if(rp == 1 && blk == 0){ + // sb_acc_2 = svmmla_s32(sb_acc_0, q4[0], q8[0]); + // sb_acc_2 = svmmla_s32(sb_acc_0, q4[1], q8[1]); + // sb_acc_2 = svmmla_s32(sb_acc_0, q4[2], q8[2]); + // sb_acc_2 = svmmla_s32(sb_acc_0, q4[3], q8[3]); + // } + // if(rp == 1 && blk == 1){ + // sb_acc_3 = svmmla_s32(sb_acc_0, q4[0], q8[0]); + // sb_acc_3 = svmmla_s32(sb_acc_0, q4[1], q8[1]); + // sb_acc_3 = svmmla_s32(sb_acc_0, q4[2], q8[2]); + // sb_acc_3 = svmmla_s32(sb_acc_0, q4[3], q8[3]); + // } + // // sb_acc[2 * rp + blk] = acc; + // } + // } + + // Calculates the Qs muladd of every row pair (rp) rows 01 and 23 of q8 + //Low nibbles of q4 and first 4 bytes of row 01 + sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_00, q8_qs_01_0); + sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_01, q8_qs_01_1); + sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_02, q8_qs_01_2); + sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_03, q8_qs_01_3); + + //High nibbles of q4 and next 4 bytes of row 01 + sb_acc_1 = svmmla_s32(sb_acc_0, q4_nibbles_10, q8_qs_01_4); + sb_acc_1 = svmmla_s32(sb_acc_0, q4_nibbles_11, q8_qs_01_5); + sb_acc_1 = svmmla_s32(sb_acc_0, q4_nibbles_12, q8_qs_01_6); + sb_acc_1 = svmmla_s32(sb_acc_0, q4_nibbles_13, q8_qs_01_7); + + //Low nibbles of q4 and first 4 bytes of row 23 + sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_00, q8_qs_23_0); + sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_01, q8_qs_23_1); + sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_02, q8_qs_23_2); + sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_03, q8_qs_23_3); + + //High nibbles of q4 and next 4 bytes of row 23 + sb_acc_1 = svmmla_s32(sb_acc_0, q4_nibbles_10, q8_qs_23_4); + sb_acc_1 = svmmla_s32(sb_acc_0, q4_nibbles_11, q8_qs_23_5); + sb_acc_1 = svmmla_s32(sb_acc_0, q4_nibbles_12, q8_qs_23_6); + sb_acc_1 = svmmla_s32(sb_acc_0, q4_nibbles_13, q8_qs_23_7); // Scales[i] corresponds to column i const int scale_offset = cp * 2; From e8495ee41bcdc398e8db8c78679529c5bb9d7060 Mon Sep 17 00:00:00 2001 From: Anubhav Srivastava Date: Tue, 19 May 2026 13:52:43 +0530 Subject: [PATCH 05/19] SVE code for bias_acc done. Only reorder_acc remains now --- ggml/src/ggml-cpu/arch/arm/repack.cpp | 79 +++++++++++++++++++-------- 1 file changed, 56 insertions(+), 23 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cpp index 969fdfc409c9..458f8b53b670 100644 --- a/ggml/src/ggml-cpu/arch/arm/repack.cpp +++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp @@ -4229,10 +4229,10 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, // q8_qs_23[i] = svld1_s8(pg_b8_vl16, q8_base + offset + 16); // } - const svint8_t q8s[2][8] = { - { q8_qs_01[0], q8_qs_01[1], q8_qs_01[2], q8_qs_01[3], q8_qs_01[4], q8_qs_01[5], q8_qs_01[6], q8_qs_01[7] }, - { q8_qs_23[0], q8_qs_23[1], q8_qs_23[2], q8_qs_23[3], q8_qs_23[4], q8_qs_23[5], q8_qs_23[6], q8_qs_23[7] }, - }; + // const svint8_t q8s[2][8] = { + // { q8_qs_01[0], q8_qs_01[1], q8_qs_01[2], q8_qs_01[3], q8_qs_01[4], q8_qs_01[5], q8_qs_01[6], q8_qs_01[7] }, + // { q8_qs_23[0], q8_qs_23[1], q8_qs_23[2], q8_qs_23[3], q8_qs_23[4], q8_qs_23[5], q8_qs_23[6], q8_qs_23[7] }, + // }; // Q4s columns iterated in pairs (01, 23, 45, 67) for (int cp = 0; cp < ncols_interleaved / 2; cp++) { @@ -4385,28 +4385,61 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, } // Multiply Acc bsum + mins - for (int q8_row = 0; q8_row < 4; q8_row++) { - // Each pair of subblocks share the same bsums - // Load scalar bsum → broadcast to a vector (vdupq_n_s16(s)). - svint16_t bsums_vec_lo = svdup_n_s16(bsums_arr[sb][q8_row * 2]); - svint16_t bsums_vec_hi = svdup_n_s16(bsums_arr[sb][q8_row * 2 + 1]); - - bias_acc[2 * q8_row] = - vmlal_s16(bias_acc[2 * q8_row], bsums_vec_lo, vget_low_s16(q4sb_mins[0])); - bias_acc[2 * q8_row] = - vmlal_s16(bias_acc[2 * q8_row], bsums_vec_hi, vget_low_s16(q4sb_mins[1])); - bias_acc[2 * q8_row + 1] = - vmlal_s16(bias_acc[2 * q8_row + 1], bsums_vec_lo, vget_high_s16(q4sb_mins[0])); - bias_acc[2 * q8_row + 1] = - vmlal_s16(bias_acc[2 * q8_row + 1], bsums_vec_hi, vget_high_s16(q4sb_mins[1])); - } + // for (int q8_row = 0; q8_row < 4; q8_row++) { + // // Each pair of subblocks share the same bsums + // // Load scalar bsum → broadcast to a vector (vdupq_n_s16(s)). + // svint16_t bsums_vec_lo = svdup_n_s16(bsums_arr[sb][q8_row * 2]); + // svint16_t bsums_vec_hi = svdup_n_s16(bsums_arr[sb][q8_row * 2 + 1]); + + // bias_acc[2 * q8_row] = + // vmlal_s16(bias_acc[2 * q8_row], bsums_vec_lo, vget_low_s16(q4sb_mins[0])); + // bias_acc[2 * q8_row] = + // vmlal_s16(bias_acc[2 * q8_row], bsums_vec_hi, vget_low_s16(q4sb_mins[1])); + // bias_acc[2 * q8_row + 1] = + // vmlal_s16(bias_acc[2 * q8_row + 1], bsums_vec_lo, vget_high_s16(q4sb_mins[0])); + // bias_acc[2 * q8_row + 1] = + // vmlal_s16(bias_acc[2 * q8_row + 1], bsums_vec_hi, vget_high_s16(q4sb_mins[1])); + // } + + + svbool_t pg_s32 = svptrue_pat_b32(SV_VL4); + bias_acc_00 = svmla_s32_x(pg_s32, bias_acc_00, svdup_n_s32((int32_t)bsums_arr[sb][0]), svunpklo_s32(q4sb_mins_0)); + bias_acc_00 = svmla_s32_x(pg_s32, bias_acc_00, svdup_n_s32((int32_t)bsums_arr[sb][1]), svunpklo_s32(q4sb_mins_1)); + bias_acc_11 = svmla_s32_x(pg_s32, bias_acc_11, svdup_n_s32((int32_t)bsums_arr[sb][0]), svunpkhi_s32(q4sb_mins_0)); + bias_acc_11 = svmla_s32_x(pg_s32, bias_acc_11, svdup_n_s32((int32_t)bsums_arr[sb][1]), svunpkhi_s32(q4sb_mins_1)); + bias_acc_22 = svmla_s32_x(pg_s32, bias_acc_22, svdup_n_s32((int32_t)bsums_arr[sb][2]), svunpklo_s32(q4sb_mins_0)); + bias_acc_22 = svmla_s32_x(pg_s32, bias_acc_22, svdup_n_s32((int32_t)bsums_arr[sb][3]), svunpklo_s32(q4sb_mins_1)); + bias_acc_33 = svmla_s32_x(pg_s32, bias_acc_33, svdup_n_s32((int32_t)bsums_arr[sb][2]), svunpkhi_s32(q4sb_mins_0)); + bias_acc_33 = svmla_s32_x(pg_s32, bias_acc_33, svdup_n_s32((int32_t)bsums_arr[sb][3]), svunpkhi_s32(q4sb_mins_1)); + bias_acc_44 = svmla_s32_x(pg_s32, bias_acc_44, svdup_n_s32((int32_t)bsums_arr[sb][4]), svunpklo_s32(q4sb_mins_0)); + bias_acc_44 = svmla_s32_x(pg_s32, bias_acc_44, svdup_n_s32((int32_t)bsums_arr[sb][5]), svunpklo_s32(q4sb_mins_1)); + bias_acc_55 = svmla_s32_x(pg_s32, bias_acc_55, svdup_n_s32((int32_t)bsums_arr[sb][4]), svunpklo_s32(q4sb_mins_0)); + bias_acc_55 = svmla_s32_x(pg_s32, bias_acc_55, svdup_n_s32((int32_t)bsums_arr[sb][5]), svunpklo_s32(q4sb_mins_1)); + bias_acc_66 = svmla_s32_x(pg_s32, bias_acc_66, svdup_n_s32((int32_t)bsums_arr[sb][6]), svunpklo_s32(q4sb_mins_0)); + bias_acc_66 = svmla_s32_x(pg_s32, bias_acc_66, svdup_n_s32((int32_t)bsums_arr[sb][7]), svunpklo_s32(q4sb_mins_1)); + bias_acc_77 = svmla_s32_x(pg_s32, bias_acc_77, svdup_n_s32((int32_t)bsums_arr[sb][6]), svunpklo_s32(q4sb_mins_0)); + bias_acc_77 = svmla_s32_x(pg_s32, bias_acc_77, svdup_n_s32((int32_t)bsums_arr[sb][7]), svunpklo_s32(q4sb_mins_1)); } // for sb + // // Reorder of i8mm output with bias and output layout + // for (int i = 0; i < 8; i++) { + // int32x2x2_t aux = vzip_s32(vget_low_s32(acc[i]), vget_high_s32(acc[i])); + // acc[i] = vcombine_s32(aux.val[0], aux.val[1]); + // } + // Reorder of i8mm output with bias and output layout - for (int i = 0; i < 8; i++) { - int32x2x2_t aux = vzip_s32(vget_low_s32(acc[i]), vget_high_s32(acc[i])); - acc[i] = vcombine_s32(aux.val[0], aux.val[1]); - } + const uint32_t perm_arr[4] = { 0, 2, 1, 3 }; + svuint32_t perm = svld1_u32(svptrue_pat_b32(SV_VL4), perm_arr); + acc_00 = svtbl_s32(acc_00, perm); + acc_11 = svtbl_s32(acc_11, perm); + acc_22 = svtbl_s32(acc_22, perm); + acc_33 = svtbl_s32(acc_33, perm); + acc_44 = svtbl_s32(acc_44, perm); + acc_55 = svtbl_s32(acc_55, perm); + acc_66 = svtbl_s32(acc_66, perm); + acc_77 = svtbl_s32(acc_77, perm); + + int32x4_t reorder_acc[8] = { vcombine_s32(vget_low_s32(acc[0]), vget_low_s32(acc[1])), vcombine_s32(vget_low_s32(acc[2]), vget_low_s32(acc[3])), From c124bcd0e20e450bea718c2d93cd3d3d3ac867ca Mon Sep 17 00:00:00 2001 From: Anubhav Srivastava Date: Tue, 19 May 2026 14:32:17 +0530 Subject: [PATCH 06/19] reorder_acc converted to SVE --- ggml/src/ggml-cpu/arch/arm/repack.cpp | 30 ++++++++++++++++++--------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cpp index 458f8b53b670..d9fe340d2efe 100644 --- a/ggml/src/ggml-cpu/arch/arm/repack.cpp +++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp @@ -4440,16 +4440,26 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, acc_77 = svtbl_s32(acc_77, perm); - int32x4_t reorder_acc[8] = { - vcombine_s32(vget_low_s32(acc[0]), vget_low_s32(acc[1])), - vcombine_s32(vget_low_s32(acc[2]), vget_low_s32(acc[3])), - vcombine_s32(vget_high_s32(acc[0]), vget_high_s32(acc[1])), - vcombine_s32(vget_high_s32(acc[2]), vget_high_s32(acc[3])), - vcombine_s32(vget_low_s32(acc[4]), vget_low_s32(acc[5])), - vcombine_s32(vget_low_s32(acc[6]), vget_low_s32(acc[7])), - vcombine_s32(vget_high_s32(acc[4]), vget_high_s32(acc[5])), - vcombine_s32(vget_high_s32(acc[6]), vget_high_s32(acc[7])), - }; + // int32x4_t reorder_acc[8] = { + // vcombine_s32(vget_low_s32(acc[0]), vget_low_s32(acc[1])), + // vcombine_s32(vget_low_s32(acc[2]), vget_low_s32(acc[3])), + // vcombine_s32(vget_high_s32(acc[0]), vget_high_s32(acc[1])), + // vcombine_s32(vget_high_s32(acc[2]), vget_high_s32(acc[3])), + // vcombine_s32(vget_low_s32(acc[4]), vget_low_s32(acc[5])), + // vcombine_s32(vget_low_s32(acc[6]), vget_low_s32(acc[7])), + // vcombine_s32(vget_high_s32(acc[4]), vget_high_s32(acc[5])), + // vcombine_s32(vget_high_s32(acc[6]), vget_high_s32(acc[7])), + // }; + + reorder_acc_0 = svsplice_s32(svptrue_pat_b32(SV_VL2), acc_00, acc_11); + reorder_acc_1 = svsplice_s32(svptrue_pat_b32(SV_VL2), acc_22, acc_33); + reorder_acc_2 = svsplice_s32(svptrue_pat_b32(SV_VL2), svext_s32(acc_00, acc_00, 2), svext_s32(acc_11, acc_11, 2)); + reorder_acc_3 = svsplice_s32(svptrue_pat_b32(SV_VL2), svext_s32(acc_22, acc_22, 2), svext_s32(acc_33, acc_33, 2)); + reorder_acc_4 = svsplice_s32(svptrue_pat_b32(SV_VL2), acc[4], acc[5]) + reorder_acc_5 = svsplice_s32(svptrue_pat_b32(SV_VL2), acc[6], acc[7]) + reorder_acc_6 = svsplice_s32(svptrue_pat_b32(SV_VL2), svext_s32(acc_44, acc_44, 2), svext_s32(acc_55, acc_55, 2)); + reorder_acc_7 = svsplice_s32(svptrue_pat_b32(SV_VL2), svext_s32(acc_66, acc_66, 2), svext_s32(acc_77, acc_77, 2)); + for (int i = 0; i < q8_k_blocklen; i++) { for (int j = 0; j < 2; j++) { From a6119ad0230397d37989cfdd2f5b965a6d9b0b65 Mon Sep 17 00:00:00 2001 From: Anubhav Srivastava Date: Tue, 19 May 2026 14:38:13 +0530 Subject: [PATCH 07/19] datatypes added for reorder_acc --- ggml/src/ggml-cpu/arch/arm/repack.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cpp index d9fe340d2efe..64d625a2e6f6 100644 --- a/ggml/src/ggml-cpu/arch/arm/repack.cpp +++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp @@ -4451,14 +4451,14 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, // vcombine_s32(vget_high_s32(acc[6]), vget_high_s32(acc[7])), // }; - reorder_acc_0 = svsplice_s32(svptrue_pat_b32(SV_VL2), acc_00, acc_11); - reorder_acc_1 = svsplice_s32(svptrue_pat_b32(SV_VL2), acc_22, acc_33); - reorder_acc_2 = svsplice_s32(svptrue_pat_b32(SV_VL2), svext_s32(acc_00, acc_00, 2), svext_s32(acc_11, acc_11, 2)); - reorder_acc_3 = svsplice_s32(svptrue_pat_b32(SV_VL2), svext_s32(acc_22, acc_22, 2), svext_s32(acc_33, acc_33, 2)); - reorder_acc_4 = svsplice_s32(svptrue_pat_b32(SV_VL2), acc[4], acc[5]) - reorder_acc_5 = svsplice_s32(svptrue_pat_b32(SV_VL2), acc[6], acc[7]) - reorder_acc_6 = svsplice_s32(svptrue_pat_b32(SV_VL2), svext_s32(acc_44, acc_44, 2), svext_s32(acc_55, acc_55, 2)); - reorder_acc_7 = svsplice_s32(svptrue_pat_b32(SV_VL2), svext_s32(acc_66, acc_66, 2), svext_s32(acc_77, acc_77, 2)); + svint32_t reorder_acc_0 = svsplice_s32(svptrue_pat_b32(SV_VL2), acc_00, acc_11); + svint32_t reorder_acc_1 = svsplice_s32(svptrue_pat_b32(SV_VL2), acc_22, acc_33); + svint32_t reorder_acc_2 = svsplice_s32(svptrue_pat_b32(SV_VL2), svext_s32(acc_00, acc_00, 2), svext_s32(acc_11, acc_11, 2)); + svint32_t reorder_acc_3 = svsplice_s32(svptrue_pat_b32(SV_VL2), svext_s32(acc_22, acc_22, 2), svext_s32(acc_33, acc_33, 2)); + svint32_t reorder_acc_4 = svsplice_s32(svptrue_pat_b32(SV_VL2), acc_44, acc_55); + svint32_t reorder_acc_5 = svsplice_s32(svptrue_pat_b32(SV_VL2), acc_66, acc_77); + svint32_t reorder_acc_6 = svsplice_s32(svptrue_pat_b32(SV_VL2), svext_s32(acc_44, acc_44, 2), svext_s32(acc_55, acc_55, 2)); + svint32_t reorder_acc_7 = svsplice_s32(svptrue_pat_b32(SV_VL2), svext_s32(acc_66, acc_66, 2), svext_s32(acc_77, acc_77, 2)); for (int i = 0; i < q8_k_blocklen; i++) { From 88ca4b89dd43da07f32fb8db54879734a54651ac Mon Sep 17 00:00:00 2001 From: Anubhav Srivastava Date: Wed, 20 May 2026 14:44:50 +0530 Subject: [PATCH 08/19] SVE conversion complete - compilation passes --- ggml/src/ggml-cpu/arch/arm/repack.cpp | 255 ++++++++++++++++++++++++-- 1 file changed, 242 insertions(+), 13 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cpp index 64d625a2e6f6..976b98b60730 100644 --- a/ggml/src/ggml-cpu/arch/arm/repack.cpp +++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp @@ -4137,6 +4137,16 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, // acc_f32[i] = svdup_n_f32(0); // } + svfloat32_t acc_f32_00 = svdup_n_f32(0); + svfloat32_t acc_f32_11 = svdup_n_f32(0); + svfloat32_t acc_f32_22 = svdup_n_f32(0); + svfloat32_t acc_f32_33 = svdup_n_f32(0); + svfloat32_t acc_f32_44 = svdup_n_f32(0); + svfloat32_t acc_f32_55 = svdup_n_f32(0); + svfloat32_t acc_f32_66 = svdup_n_f32(0); + svfloat32_t acc_f32_77 = svdup_n_f32(0); + + for (int b = 0; b < nb; b++) { // bsums pairs belongs to the same q8_k subblock const int16x8_t bsums[4]{ @@ -4461,29 +4471,248 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, svint32_t reorder_acc_7 = svsplice_s32(svptrue_pat_b32(SV_VL2), svext_s32(acc_66, acc_66, 2), svext_s32(acc_77, acc_77, 2)); - for (int i = 0; i < q8_k_blocklen; i++) { - for (int j = 0; j < 2; j++) { - float32x4_t q8_d = vdupq_n_f32(q8_ptr[b].d[i]); - float32x4_t q4_dmin = vcvt_f32_f16(vld1_f16((const __fp16 *) (q4_ptr[b].dmin + j * 4))); - const float32x4_t dmins = vmulq_f32(q4_dmin, q8_d); + // for (int i = 0; i < q8_k_blocklen; i++) { + // for (int j = 0; j < 2; j++) { + // float32x4_t q8_d = vdupq_n_f32(q8_ptr[b].d[i]); + // float32x4_t q4_dmin = vcvt_f32_f16(vld1_f16((const __fp16 *) (q4_ptr[b].dmin + j * 4))); + // const float32x4_t dmins = vmulq_f32(q4_dmin, q8_d); - float32x4_t q4_d = vcvt_f32_f16(vld1_f16((const __fp16 *) (q4_ptr[b].d + j * 4))); - const float32x4_t scale = vmulq_f32(q4_d, q8_d); + // float32x4_t q4_d = vcvt_f32_f16(vld1_f16((const __fp16 *) (q4_ptr[b].d + j * 4))); + // const float32x4_t scale = vmulq_f32(q4_d, q8_d); - acc_f32[2 * i + j] = vmlsq_f32(acc_f32[2 * i + j], vcvtq_f32_s32(bias_acc[2 * i + j]), dmins); - acc_f32[2 * i + j] = - vmlaq_f32(acc_f32[2 * i + j], vcvtq_f32_s32(reorder_acc[2 * i + j]), scale); - } - } + // acc_f32[2 * i + j] = vmlsq_f32(acc_f32[2 * i + j], vcvtq_f32_s32(bias_acc[2 * i + j]), dmins); + // acc_f32[2 * i + j] = + // vmlaq_f32(acc_f32[2 * i + j], vcvtq_f32_s32(reorder_acc[2 * i + j]), scale); + // } + // } + + // i=0, j=0 + svfloat32_t q8_d = svdup_n_f32(q8_ptr[b].d[0]); + svfloat32_t q4_dmin = svcvt_f32_f16_z( + svwhilelt_b32((uint64_t)0, (uint64_t)4), + svld1_f16( + svwhilelt_b16((uint64_t)0, (uint64_t)4), + (const __fp16 *) (q4_ptr[b].dmin + 0 * 4) + ) + ); + svfloat32_t dmins = svmul_f32_x(svptrue_b32(), q4_dmin, q8_d); + + svfloat32_t q4_d = svcvt_f32_f16_z( + svwhilelt_b32((uint64_t)0, (uint64_t)4), + svld1_f16( + svwhilelt_b16((uint64_t)0, (uint64_t)4), + (const __fp16 *) (q4_ptr[b].d + 0 * 4) + ) + ); + svfloat32_t scale = svmul_f32_x(svptrue_b32(), q4_d, q8_d); + + acc_f32_00 = svmls_f32_m(svptrue_b32(), acc_f32_00, svcvt_f32_s32_x(svptrue_b32(), bias_acc_00), dmins); + acc_f32_00 = svmla_f32_m(svptrue_b32(), acc_f32_00, svcvt_f32_s32_x(svptrue_b32(), reorder_acc_0), scale); + + //i == 0, j == 1 + q8_d = svdup_n_f32(q8_ptr[b].d[0]); + q4_dmin = svcvt_f32_f16_z( + svwhilelt_b32((uint64_t)0, (uint64_t)4), + svld1_f16( + svwhilelt_b16((uint64_t)0, (uint64_t)4), + (const __fp16 *) (q4_ptr[b].dmin + 1 * 4) + ) + ); + dmins = svmul_f32_x(svptrue_b32(), q4_dmin, q8_d); + + q4_d = svcvt_f32_f16_z( + svwhilelt_b32((uint64_t)0, (uint64_t)4), + svld1_f16( + svwhilelt_b16((uint64_t)0, (uint64_t)4), + (const __fp16 *) (q4_ptr[b].d + 1 * 4) + ) + ); + scale = svmul_f32_x(svptrue_b32(), q4_d, q8_d); + + acc_f32_11 = svmls_f32_m(svptrue_b32(), acc_f32_11, svcvt_f32_s32_x(svptrue_b32(), bias_acc_11), dmins); + acc_f32_11 = svmla_f32_m(svptrue_b32(), acc_f32_11, svcvt_f32_s32_x(svptrue_b32(), reorder_acc_1), scale); + + //i == 1, j == 0 + q8_d = svdup_n_f32(q8_ptr[b].d[1]); + q4_dmin = svcvt_f32_f16_z( + svwhilelt_b32((uint64_t)0, (uint64_t)4), + svld1_f16( + svwhilelt_b16((uint64_t)0, (uint64_t)4), + (const __fp16 *) (q4_ptr[b].dmin + 0 * 4) + ) + ); + dmins = svmul_f32_x(svptrue_b32(), q4_dmin, q8_d); + + q4_d = svcvt_f32_f16_z( + svwhilelt_b32((uint64_t)0, (uint64_t)4), + svld1_f16( + svwhilelt_b16((uint64_t)0, (uint64_t)4), + (const __fp16 *) (q4_ptr[b].d + 0 * 4) + ) + ); + scale = svmul_f32_x(svptrue_b32(), q4_d, q8_d); + + acc_f32_22 = svmls_f32_m(svptrue_b32(), acc_f32_22, svcvt_f32_s32_x(svptrue_b32(), bias_acc_22), dmins); + acc_f32_22 = svmla_f32_m(svptrue_b32(), acc_f32_22, svcvt_f32_s32_x(svptrue_b32(), reorder_acc_2), scale); + + //i == 1, j == 1 + q8_d = svdup_n_f32(q8_ptr[b].d[1]); + q4_dmin = svcvt_f32_f16_z( + svwhilelt_b32((uint64_t)0, (uint64_t)4), + svld1_f16( + svwhilelt_b16((uint64_t)0, (uint64_t)4), + (const __fp16 *) (q4_ptr[b].dmin + 1 * 4) + ) + ); + dmins = svmul_f32_x(svptrue_b32(), q4_dmin, q8_d); + + q4_d = svcvt_f32_f16_z( + svwhilelt_b32((uint64_t)0, (uint64_t)4), + svld1_f16( + svwhilelt_b16((uint64_t)0, (uint64_t)4), + (const __fp16 *) (q4_ptr[b].d + 1 * 4) + ) + ); + scale = svmul_f32_x(svptrue_b32(), q4_d, q8_d); + + acc_f32_33 = svmls_f32_m(svptrue_b32(), acc_f32_33, svcvt_f32_s32_x(svptrue_b32(), bias_acc_33), dmins); + acc_f32_33 = svmla_f32_m(svptrue_b32(), acc_f32_33, svcvt_f32_s32_x(svptrue_b32(), reorder_acc_3), scale); + + //i == 2, j == 0 + q8_d = svdup_n_f32(q8_ptr[b].d[2]); + q4_dmin = svcvt_f32_f16_z( + svwhilelt_b32((uint64_t)0, (uint64_t)4), + svld1_f16( + svwhilelt_b16((uint64_t)0, (uint64_t)4), + (const __fp16 *) (q4_ptr[b].dmin + 0 * 4) + ) + ); + dmins = svmul_f32_x(svptrue_b32(), q4_dmin, q8_d); + + q4_d = svcvt_f32_f16_z( + svwhilelt_b32((uint64_t)0, (uint64_t)4), + svld1_f16( + svwhilelt_b16((uint64_t)0, (uint64_t)4), + (const __fp16 *) (q4_ptr[b].d + 0 * 4) + ) + ); + scale = svmul_f32_x(svptrue_b32(), q4_d, q8_d); + + acc_f32_44 = svmls_f32_m(svptrue_b32(), acc_f32_44, svcvt_f32_s32_x(svptrue_b32(), bias_acc_44), dmins); + acc_f32_44 = svmla_f32_m(svptrue_b32(), acc_f32_44, svcvt_f32_s32_x(svptrue_b32(), reorder_acc_4), scale); + + //i == 2, j == 1 + q8_d = svdup_n_f32(q8_ptr[b].d[2]); + q4_dmin = svcvt_f32_f16_z( + svwhilelt_b32((uint64_t)0, (uint64_t)4), + svld1_f16( + svwhilelt_b16((uint64_t)0, (uint64_t)4), + (const __fp16 *) (q4_ptr[b].dmin + 1 * 4) + ) + ); + dmins = svmul_f32_x(svptrue_b32(), q4_dmin, q8_d); + + q4_d = svcvt_f32_f16_z( + svwhilelt_b32((uint64_t)0, (uint64_t)4), + svld1_f16( + svwhilelt_b16((uint64_t)0, (uint64_t)4), + (const __fp16 *) (q4_ptr[b].d + 1 * 4) + ) + ); + scale = svmul_f32_x(svptrue_b32(), q4_d, q8_d); + + acc_f32_55 = svmls_f32_m(svptrue_b32(), acc_f32_55, svcvt_f32_s32_x(svptrue_b32(), bias_acc_55), dmins); + acc_f32_55 = svmla_f32_m(svptrue_b32(), acc_f32_55, svcvt_f32_s32_x(svptrue_b32(), reorder_acc_5), scale); + + //i == 3, j == 0 + q8_d = svdup_n_f32(q8_ptr[b].d[3]); + q4_dmin = svcvt_f32_f16_z( + svwhilelt_b32((uint64_t)0, (uint64_t)4), + svld1_f16( + svwhilelt_b16((uint64_t)0, (uint64_t)4), + (const __fp16 *) (q4_ptr[b].dmin + 0 * 4) + ) + ); + dmins = svmul_f32_x(svptrue_b32(), q4_dmin, q8_d); + + q4_d = svcvt_f32_f16_z( + svwhilelt_b32((uint64_t)0, (uint64_t)4), + svld1_f16( + svwhilelt_b16((uint64_t)0, (uint64_t)4), + (const __fp16 *) (q4_ptr[b].d + 0 * 4) + ) + ); + scale = svmul_f32_x(svptrue_b32(), q4_d, q8_d); + + acc_f32_66 = svmls_f32_m(svptrue_b32(), acc_f32_66, svcvt_f32_s32_x(svptrue_b32(), bias_acc_66), dmins); + acc_f32_66 = svmla_f32_m(svptrue_b32(), acc_f32_66, svcvt_f32_s32_x(svptrue_b32(), reorder_acc_6), scale); + + //i == 3, j == 1 + q8_d = svdup_n_f32(q8_ptr[b].d[3]); + q4_dmin = svcvt_f32_f16_z( + svwhilelt_b32((uint64_t)0, (uint64_t)4), + svld1_f16( + svwhilelt_b16((uint64_t)0, (uint64_t)4), + (const __fp16 *) (q4_ptr[b].dmin + 1 * 4) + ) + ); + dmins = svmul_f32_x(svptrue_b32(), q4_dmin, q8_d); + + q4_d = svcvt_f32_f16_z( + svwhilelt_b32((uint64_t)0, (uint64_t)4), + svld1_f16( + svwhilelt_b16((uint64_t)0, (uint64_t)4), + (const __fp16 *) (q4_ptr[b].d + 1 * 4) + ) + ); + scale = svmul_f32_x(svptrue_b32(), q4_d, q8_d); + + acc_f32_77 = svmls_f32_m(svptrue_b32(), acc_f32_77, svcvt_f32_s32_x(svptrue_b32(), bias_acc_77), dmins); + acc_f32_77 = svmla_f32_m(svptrue_b32(), acc_f32_77, svcvt_f32_s32_x(svptrue_b32(), reorder_acc_7), scale); } // for b // With the previous reorder, the tile is already in the correct memory layout. + // for (int i = 0; i < q8_k_blocklen; i++) { + // int row = y * q8_k_blocklen + i; + // for (int j = 0; j < 2; j++) { + // int col = x * ncols_interleaved + j * 4; + // int offset = row * bs + col; + // vst1q_f32(s + offset, acc_f32[2 * i + j]); + // } + // } + // Predicate for exactly 4 lanes + svbool_t pg4 = svptrue_pat_b32(SV_VL4); for (int i = 0; i < q8_k_blocklen; i++) { int row = y * q8_k_blocklen + i; for (int j = 0; j < 2; j++) { int col = x * ncols_interleaved + j * 4; int offset = row * bs + col; - vst1q_f32(s + offset, acc_f32[2 * i + j]); + + if (i == 0 && j == 0) { + // acc_f32_0 → lower half of acc_f32_01 + svst1_f32(pg4, s + offset, acc_f32_00); + } else if (i == 0 && j == 1) { + // acc_f32_1 → upper half of acc_f32_01 + svst1_f32(pg4, s + offset, acc_f32_11); + } else if (i == 1 && j == 0) { + // acc_f32_2 + svst1_f32(pg4, s + offset, acc_f32_22); + } else if (i == 1 && j == 1) { + // acc_f32_3 + svst1_f32(pg4, s + offset, acc_f32_33); + } else if (i == 2 && j == 0) { + // acc_f32_4 + svst1_f32(pg4, s + offset, acc_f32_44); + } else if (i == 2 && j == 1) { + // acc_f32_5 + svst1_f32(pg4, s + offset, acc_f32_55); + } else if (i == 3 && j == 0) { + // acc_f32_6 + svst1_f32(pg4, s + offset, acc_f32_66); + } else if (i == 3 && j == 1) { + // acc_f32_7 + svst1_f32(pg4, s + offset, acc_f32_77); + } } } } // for x From e0864a7b71a665fb00e00177d656df08fc320eb8 Mon Sep 17 00:00:00 2001 From: Anubhav Srivastava Date: Wed, 20 May 2026 15:34:26 +0530 Subject: [PATCH 09/19] execution is working-commented out the cout --- ggml/src/ggml-cpu/arch/arm/repack.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cpp index 976b98b60730..0dfdc3d9f16d 100644 --- a/ggml/src/ggml-cpu/arch/arm/repack.cpp +++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp @@ -4117,7 +4117,7 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, } case 128: { - std::cout << "VL is 128" << std::endl; + //std::cout << "VL is 128" << std::endl; //constexpr int q8_k_blocklen = 4; const svuint8_t m4b = svdup_n_u8(0x0f); svbool_t pg_b16_vl8 = svptrue_pat_b16(SV_VL8); From 70cfc5a92527c93ef1580816d28dc1e9c558489d Mon Sep 17 00:00:00 2001 From: Anubhav Srivastava Date: Wed, 20 May 2026 16:32:54 +0530 Subject: [PATCH 10/19] added return instead of break in switch --- ggml/src/ggml-cpu/arch/arm/repack.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cpp index 0dfdc3d9f16d..19951f3b6caa 100644 --- a/ggml/src/ggml-cpu/arch/arm/repack.cpp +++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp @@ -4113,7 +4113,7 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, } } // for x } // for y - break; + return; } case 128: { @@ -4717,10 +4717,10 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, } } // for x } // for y - break; + return; } default: - std::cout << "Invalid VL. VL is neither 128 nor 256" << std::endl; + std::cout << "Invalid VL. VL is neither 128 nor 256" << std::endl; } #endif // SVE compile-time end From 6261739a17cc79a6b37f00d459f4b3a0bf2d14a6 Mon Sep 17 00:00:00 2001 From: Anubhav Srivastava Date: Thu, 21 May 2026 18:28:59 +0530 Subject: [PATCH 11/19] code added for debug dumps and prints - correct till q8 block loading --- ggml/src/ggml-cpu/arch/arm/repack.cpp | 220 ++++++++++++++++++++------ 1 file changed, 171 insertions(+), 49 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cpp index 19951f3b6caa..f0976e5ca914 100644 --- a/ggml/src/ggml-cpu/arch/arm/repack.cpp +++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp @@ -15,6 +15,9 @@ #include // for qsort #include // for GGML_ASSERT #include +#include +#include +#include #define GGML_CPU_CLANG_WORKAROUND #include "../../repack.h" @@ -25,6 +28,106 @@ #define UNUSED GGML_UNUSED +#include + +void dump_svint32(const std::string& filename, svint32_t vec) +{ + const std::uint64_t lanes = svcntw(); + + std::vector buffer(lanes); + + svbool_t pg = svptrue_b32(); + + svst1_s32(pg, buffer.data(), vec); + + std::ofstream out(filename); + if (!out) { + std::cerr << "Failed to open file: " << filename << '\n'; + return; + } + + for (std::uint64_t i = 0; i < lanes; ++i) { + out << "lane[" << i << "] = " << buffer[i] << '\n'; + } +} + + + +void dump_svuint32_to_file(const std::string& filename, svuint32_t vec) +{ + // Number of 32-bit lanes for the current SVE vector length + const std::uint64_t lanes = svcntw(); + + // Temporary scalar buffer + std::vector buffer(lanes); + + // Store all lanes from SVE vector into normal memory + svst1_u32(svptrue_b32(), buffer.data(), vec); + + // Create / overwrite the text file + std::ofstream out(filename); + if (!out) { + std::cerr << "Failed to create/open file: " << filename << '\n'; + return; + } + + // Dump each lane + for (std::uint64_t i = 0; i < lanes; ++i) { + out << "lane[" << i << "] = " << buffer[i] << '\n'; + } + out.close(); + std::exit(EXIT_SUCCESS); +} + + +void dump_sve_s16_to_file_and_exit(const std::string& filename, + svint16_t vec) +{ + // Number of signed 16-bit lanes in current SVE vector length + const std::uint64_t lanes = svcnth(); + + // Temporary scalar buffer + std::vector buffer(lanes); + + // Store all active SVE lanes into normal memory + svst1_s16(svptrue_b16(), buffer.data(), vec); + + // Create / overwrite file + std::ofstream out(filename); + if (!out) { + std::cerr << "Failed to open file: " << filename << '\n'; + std::exit(EXIT_FAILURE); + } + + for (std::uint64_t i = 0; i < lanes; ++i) { + out << "lane[" << i << "] = " << buffer[i] << '\n'; + std::cout << "lane[" << i << "] = " << buffer[i] << '\n'; + + } + + // Important because std::exit() does not run local destructors + out.close(); + // std::exit(EXIT_SUCCESS); +} + +#include +#include +#include + +static inline void print_svint8(const char *name, svint8_t v) +{ + size_t vl = svcntb(); // Number of int8 lanes in current SVE vector + int8_t buf[vl]; + + svst1_s8(svptrue_b8(), buf, v); + + printf("%s = [", name); + for (size_t i = 0; i < vl; ++i) { + printf("%s%d", i ? ", " : "", (int)buf[i]); + } + printf("]\n"); +} + #if defined(__aarch64__) && defined(__ARM_NEON) && (defined(__ARM_FEATURE_MATMUL_INT8) || defined(__ARM_FEATURE_DOTPROD)) // Helper for decoding scales and mins of Q4_K and Q5_K block formats static inline void decode_q_Kx8_6bit_scales(const uint8_t * scales_in, int16x8_t * out_mins, int8_t * out_scales) { @@ -50,6 +153,28 @@ static inline void decode_q_Kx8_6bit_scales(const uint8_t * scales_in, int16x8_t #endif +bool saveBytesToFile(const uint8_t* ptr, std::size_t size, const std::string& filename) +{ + std::ofstream out(filename, std::ios::binary); + if (!out) { + return false; + } + + out.write(reinterpret_cast(ptr), size); + return static_cast(out); +} + +bool loadBytesFromFile(uint8_t* ptr, std::size_t size, const std::string& filename) +{ + std::ifstream in(filename, std::ios::binary); + if (!in) { + return false; + } + + in.read(reinterpret_cast(ptr), size); + return in.gcount() == static_cast(size); +} + #if defined(__aarch64__) && defined(__ARM_FEATURE_SVE) && (defined(__ARM_FEATURE_MATMUL_INT8) || defined(__ARM_FEATURE_DOTPROD)) // Helper for decoding scales and mins of Q4_K and Q5_K block formats static inline void decode_q_Kx8_6bit_scales_sve(const uint8_t * scales_in, svint16_t * out_mins, int8_t * out_scales) { @@ -60,20 +185,42 @@ static inline void decode_q_Kx8_6bit_scales_sve(const uint8_t * scales_in, svint uint32_t sm[3]; memcpy(sm, scales_in, scales_size); + std::cout< loadedData{}; + uint8_t* loadPtr = loadedData.data(); + if (!loadBytesFromFile(loadPtr, loadedData.size(), "data.txt")) { + std::cerr << "Failed to load file\n"; + return; + } + + std::cout << "Loaded Data" << std::endl << "scales_in: "; + for (uint8_t value : loadedData) { + std::cout << static_cast(value) << " "; + } + std::cout << "\nout_mins: \n"; const uint32_t mins_0_3 = sm[1] & kmask1; const uint32_t mins_4_7 = ((sm[2] >> 4) & kmask2) | (((sm[1] >> 6) & kmask3) << 4); // const uint32x2_t mins_u32 = { mins_0_3, mins_4_7 }; uint32_t tmp_mins[2] = { mins_0_3, mins_4_7 }; - svbool_t pg2_u32 = svptrue_pat_b32(SV_VL2); // 2 x uint32_t = 64 bits + svbool_t pg2_u32 = svptrue_pat_b32(SV_VL4); // 2 x uint32_t = 64 bits svuint32_t mins_u32 = svld1_u32(pg2_u32, tmp_mins); - *out_mins = svreinterpret_s16_u16(svunpklo_u16(svreinterpret_u8_u32(mins_u32))); + dump_sve_s16_to_file_and_exit("out_mins_s16.txt", *out_mins); uint32_t scales_u32[2]; scales_u32[0] = sm[0] & kmask1; scales_u32[1] = (sm[2] & kmask2) | (((sm[0] >> 6) & kmask3) << 4); + std::cout<<"\nout_scales: "<< scales_u32[0] <<" - "<r0 0123, [1]->r0 4567, [2]->r1 0123 ... - - // for (int i = 0; i < 8; i++) { - // acc[i] = svdup_n_f32(0); - // bias_acc[i] = svdup_n_f32(0); - // } svint32_t sb_acc_0 = svdup_n_s32(0); svint32_t sb_acc_1 = svdup_n_s32(0); @@ -4199,7 +4333,6 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, // Need scales for the low and high nibbles // 2 * 12 = 24 bytes per subblock, 4 sbs -> 4 * 24 = 96 bytes total int8_t q4sb_scales[2][8]; - // svint16_t q4sb_mins[2]; // int16 as its needed for bias_acc later svint16_t q4sb_mins_0, q4sb_mins_1; // int16 as its needed for bias_acc later for (int i = 0; i < 2; i++) { const int offset = sb * 24 + i * 12; @@ -4221,6 +4354,14 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, svint8_t q8_qs_01_5 = svld1_s8(pg_b8_vl16, q8_base + 5*32); svint8_t q8_qs_01_6 = svld1_s8(pg_b8_vl16, q8_base + 6*32); svint8_t q8_qs_01_7 = svld1_s8(pg_b8_vl16, q8_base + 7*32); + print_svint8("q8_qs_01_0", q8_qs_01_0); + print_svint8("q8_qs_01_1", q8_qs_01_1); + print_svint8("q8_qs_01_2", q8_qs_01_2); + print_svint8("q8_qs_01_3", q8_qs_01_3); + print_svint8("q8_qs_01_4", q8_qs_01_4); + print_svint8("q8_qs_01_5", q8_qs_01_5); + print_svint8("q8_qs_01_6", q8_qs_01_6); + print_svint8("q8_qs_01_7", q8_qs_01_7); // svint8_t q8_qs_23[8]; svint8_t q8_qs_23_0 = svld1_s8(pg_b8_vl16, q8_base + 0*32 + 16); @@ -4232,52 +4373,33 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, svint8_t q8_qs_23_6 = svld1_s8(pg_b8_vl16, q8_base + 6*32 + 16); svint8_t q8_qs_23_7 = svld1_s8(pg_b8_vl16, q8_base + 7*32 + 16); - // // Load 32-byte per row pair, 1 subblock each time - // for (int i = 0; i < 8; i++) { - // const int offset = i * 32; // 16 for row 01, 16 for row 23 - // q8_qs_01[i] = svld1_s8(pg_b8_vl16, q8_base + offset); - // q8_qs_23[i] = svld1_s8(pg_b8_vl16, q8_base + offset + 16); - // } + print_svint8("q8_qs_23_0", q8_qs_23_0); + print_svint8("q8_qs_23_1", q8_qs_23_1); + print_svint8("q8_qs_23_2", q8_qs_23_2); + print_svint8("q8_qs_23_3", q8_qs_23_3); + print_svint8("q8_qs_23_4", q8_qs_23_4); + print_svint8("q8_qs_23_5", q8_qs_23_5); + print_svint8("q8_qs_23_6", q8_qs_23_6); + print_svint8("q8_qs_23_7", q8_qs_23_7); - // const svint8_t q8s[2][8] = { - // { q8_qs_01[0], q8_qs_01[1], q8_qs_01[2], q8_qs_01[3], q8_qs_01[4], q8_qs_01[5], q8_qs_01[6], q8_qs_01[7] }, - // { q8_qs_23[0], q8_qs_23[1], q8_qs_23[2], q8_qs_23[3], q8_qs_23[4], q8_qs_23[5], q8_qs_23[6], q8_qs_23[7] }, - // }; + std::exit(EXIT_SUCCESS); // Q4s columns iterated in pairs (01, 23, 45, 67) for (int cp = 0; cp < ncols_interleaved / 2; cp++) { - // //This is not allowed - change - // for (int i = 0; i < 4; i++) { - // sb_acc[i] = svdup_n_s32(0); - // } - - svuint8_t q4_qs_cp_0 = svld1_u8(pg_b8_vl16, q4_ptr[b].qs + sb * QK_K + 16 * cp + 0); // 0 .. 7 & 32..39 + svuint8_t q4_qs_cp_0 = svld1_u8(pg_b8_vl16, q4_ptr[b].qs + sb * QK_K + 16 * cp + 0); // 0 .. 7 & 32..39 svuint8_t q4_qs_cp_1 = svld1_u8(pg_b8_vl16, q4_ptr[b].qs + sb * QK_K + 16 * cp + 64); // 8 ..15 & 40..47 svuint8_t q4_qs_cp_2 = svld1_u8(pg_b8_vl16, q4_ptr[b].qs + sb * QK_K + 16 * cp + 128); // 16..23 & 48..55 svuint8_t q4_qs_cp_3 = svld1_u8(pg_b8_vl16, q4_ptr[b].qs + sb * QK_K + 16 * cp + 192); // 24..31 & 56..63 - // const svint8_t q4_nibbles[2][4] = { - // { - // svreinterpret_s8_u8(svand_u8_x(pg_b8_vl16, q4_qs_cp_0, m4b)), - // svreinterpret_s8_u8(svand_u8_x(pg_b8_vl16, q4_qs_cp_1, m4b)), - // svreinterpret_s8_u8(svand_u8_x(pg_b8_vl16, q4_qs_cp_2, m4b)), - // svreinterpret_s8_u8(svand_u8_x(pg_b8_vl16, q4_qs_cp_3, m4b)), - // }, - // { - // svreinterpret_s8_u8(svlsr_n_u8_x(pg_b8_vl16, q4_qs_cp_0, 4)), - // svreinterpret_s8_u8(svlsr_n_u8_x(pg_b8_vl16, q4_qs_cp_1, 4)), - // svreinterpret_s8_u8(svlsr_n_u8_x(pg_b8_vl16, q4_qs_cp_2, 4)), - // svreinterpret_s8_u8(svlsr_n_u8_x(pg_b8_vl16, q4_qs_cp_3, 4)), - // } - // }; + svint8_t q4_nibbles_00 = svreinterpret_s8_u8(svand_u8_x(pg_b8_vl16, q4_qs_cp_0, m4b)); svint8_t q4_nibbles_01 = svreinterpret_s8_u8(svand_u8_x(pg_b8_vl16, q4_qs_cp_1, m4b)); svint8_t q4_nibbles_02 = svreinterpret_s8_u8(svand_u8_x(pg_b8_vl16, q4_qs_cp_2, m4b)); svint8_t q4_nibbles_03 = svreinterpret_s8_u8(svand_u8_x(pg_b8_vl16, q4_qs_cp_3, m4b)); svint8_t q4_nibbles_10 = svreinterpret_s8_u8(svlsr_n_u8_x(pg_b8_vl16, q4_qs_cp_0, 4)); - svint8_t q4_nibbles_11 = svreinterpret_s8_u8(svlsr_n_u8_x(pg_b8_vl16, q4_qs_cp_0, 4)); - svint8_t q4_nibbles_12 = svreinterpret_s8_u8(svlsr_n_u8_x(pg_b8_vl16, q4_qs_cp_0, 4)); - svint8_t q4_nibbles_13 = svreinterpret_s8_u8(svlsr_n_u8_x(pg_b8_vl16, q4_qs_cp_0, 4)); + svint8_t q4_nibbles_11 = svreinterpret_s8_u8(svlsr_n_u8_x(pg_b8_vl16, q4_qs_cp_1, 4)); + svint8_t q4_nibbles_12 = svreinterpret_s8_u8(svlsr_n_u8_x(pg_b8_vl16, q4_qs_cp_2, 4)); + svint8_t q4_nibbles_13 = svreinterpret_s8_u8(svlsr_n_u8_x(pg_b8_vl16, q4_qs_cp_3, 4)); // Calculates the Qs muladd of every row pair (rp) rows 01 and 23 of q8 // for each of the internal 32 qs subblock (blk) From 2a2e1d7cdcf9b033da389ae229cf8c0729806f9d Mon Sep 17 00:00:00 2001 From: Anubhav Srivastava Date: Fri, 22 May 2026 20:17:22 +0530 Subject: [PATCH 12/19] Finally 128 is running - debug cout commented --- ggml/src/ggml-cpu/arch/arm/repack.cpp | 581 +++++++++++++------------- 1 file changed, 280 insertions(+), 301 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cpp index f0976e5ca914..b7695905c926 100644 --- a/ggml/src/ggml-cpu/arch/arm/repack.cpp +++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp @@ -30,29 +30,6 @@ #include -void dump_svint32(const std::string& filename, svint32_t vec) -{ - const std::uint64_t lanes = svcntw(); - - std::vector buffer(lanes); - - svbool_t pg = svptrue_b32(); - - svst1_s32(pg, buffer.data(), vec); - - std::ofstream out(filename); - if (!out) { - std::cerr << "Failed to open file: " << filename << '\n'; - return; - } - - for (std::uint64_t i = 0; i < lanes; ++i) { - out << "lane[" << i << "] = " << buffer[i] << '\n'; - } -} - - - void dump_svuint32_to_file(const std::string& filename, svuint32_t vec) { // Number of 32-bit lanes for the current SVE vector length @@ -128,6 +105,75 @@ static inline void print_svint8(const char *name, svint8_t v) printf("]\n"); } + +void print_sve_u8(const char* label, svuint8_t v) +{ + // Number of 8‑bit lanes for the current SVE implementation + const int vl = svcntb(); + + // Stack buffer (GCC/Clang extension; fine for debug) + alignas(64) uint8_t buf[svcntb()]; + + // Predicate: all lanes active + svbool_t pg = svptrue_b8(); + + // Store vector to memory + svst1(pg, buf, v); + + // Print + std::cout << label << " (vl=" << vl << "): "; + for (int i = 0; i < vl; ++i) { + std::cout << static_cast(buf[i]) << ' '; + } + std::cout << '\n'; +} + + +void print_sve_s32(const char* label, svint32_t v) +{ + // Number of 32‑bit lanes for the current SVE implementation + const int vl = svcntw(); + + // Stack buffer sized at runtime (allowed in C++, but GCC/Clang extension) + alignas(64) int32_t buf[svcntw()]; + + // Predicate: all lanes active + svbool_t pg = svptrue_b32(); + + // Store vector to memory + svst1(pg, buf, v); + + // Print + std::cout << label << " (vl=" << vl << "): "; + for (int i = 0; i < vl; ++i) { + std::cout << buf[i] << ' '; + } + std::cout << '\n'; +} + +void print_sve_f32(const char* label, svfloat32_t v) +{ + // Number of 32‑bit floating‑point lanes + const int vl = svcntw(); + + // Portable C++ buffer + std::vector buf(vl); + + // Predicate: all lanes active + svbool_t pg = svptrue_b32(); + + // Store SVE vector to memory + svst1(pg, buf.data(), v); + + // Print + std::cout << label << " (vl=" << vl << "): "; + for (float x : buf) { + std::cout << x << ' '; + } + std::cout << '\n'; +} + + #if defined(__aarch64__) && defined(__ARM_NEON) && (defined(__ARM_FEATURE_MATMUL_INT8) || defined(__ARM_FEATURE_DOTPROD)) // Helper for decoding scales and mins of Q4_K and Q5_K block formats static inline void decode_q_Kx8_6bit_scales(const uint8_t * scales_in, int16x8_t * out_mins, int8_t * out_scales) { @@ -185,27 +231,27 @@ static inline void decode_q_Kx8_6bit_scales_sve(const uint8_t * scales_in, svint uint32_t sm[3]; memcpy(sm, scales_in, scales_size); - std::cout< loadedData{}; - uint8_t* loadPtr = loadedData.data(); - - if (!loadBytesFromFile(loadPtr, loadedData.size(), "data.txt")) { - std::cerr << "Failed to load file\n"; - return; - } - - std::cout << "Loaded Data" << std::endl << "scales_in: "; - for (uint8_t value : loadedData) { - std::cout << static_cast(value) << " "; - } - std::cout << "\nout_mins: \n"; + // std::array loadedData{}; + // uint8_t* loadPtr = loadedData.data(); + + // if (!loadBytesFromFile(loadPtr, loadedData.size(), "data.txt")) { + // std::cerr << "Failed to load file\n"; + // return; + // } + + // std::cout << "Loaded Data" << std::endl << "scales_in: "; + // for (uint8_t value : loadedData) { + // std::cout << static_cast(value) << " "; + // } + // std::cout << "\nout_mins: \n"; const uint32_t mins_0_3 = sm[1] & kmask1; const uint32_t mins_4_7 = ((sm[2] >> 4) & kmask2) | (((sm[1] >> 6) & kmask3) << 4); // const uint32x2_t mins_u32 = { mins_0_3, mins_4_7 }; @@ -214,12 +260,12 @@ static inline void decode_q_Kx8_6bit_scales_sve(const uint8_t * scales_in, svint svuint32_t mins_u32 = svld1_u32(pg2_u32, tmp_mins); *out_mins = svreinterpret_s16_u16(svunpklo_u16(svreinterpret_u8_u32(mins_u32))); - dump_sve_s16_to_file_and_exit("out_mins_s16.txt", *out_mins); + // dump_sve_s16_to_file_and_exit("out_mins_s16.txt", *out_mins); uint32_t scales_u32[2]; scales_u32[0] = sm[0] & kmask1; scales_u32[1] = (sm[2] & kmask2) | (((sm[0] >> 6) & kmask3) << 4); - std::cout<<"\nout_scales: "<< scales_u32[0] <<" - "< Date: Tue, 2 Jun 2026 13:03:36 +0530 Subject: [PATCH 13/19] Debug log cleanup --- ggml/src/ggml-cpu/arch/arm/repack.cpp | 368 +------------------------- 1 file changed, 7 insertions(+), 361 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cpp index b7695905c926..c3f63775636a 100644 --- a/ggml/src/ggml-cpu/arch/arm/repack.cpp +++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp @@ -15,9 +15,6 @@ #include // for qsort #include // for GGML_ASSERT #include -#include -#include -#include #define GGML_CPU_CLANG_WORKAROUND #include "../../repack.h" @@ -28,152 +25,6 @@ #define UNUSED GGML_UNUSED -#include - -void dump_svuint32_to_file(const std::string& filename, svuint32_t vec) -{ - // Number of 32-bit lanes for the current SVE vector length - const std::uint64_t lanes = svcntw(); - - // Temporary scalar buffer - std::vector buffer(lanes); - - // Store all lanes from SVE vector into normal memory - svst1_u32(svptrue_b32(), buffer.data(), vec); - - // Create / overwrite the text file - std::ofstream out(filename); - if (!out) { - std::cerr << "Failed to create/open file: " << filename << '\n'; - return; - } - - // Dump each lane - for (std::uint64_t i = 0; i < lanes; ++i) { - out << "lane[" << i << "] = " << buffer[i] << '\n'; - } - out.close(); - std::exit(EXIT_SUCCESS); -} - - -void dump_sve_s16_to_file_and_exit(const std::string& filename, - svint16_t vec) -{ - // Number of signed 16-bit lanes in current SVE vector length - const std::uint64_t lanes = svcnth(); - - // Temporary scalar buffer - std::vector buffer(lanes); - - // Store all active SVE lanes into normal memory - svst1_s16(svptrue_b16(), buffer.data(), vec); - - // Create / overwrite file - std::ofstream out(filename); - if (!out) { - std::cerr << "Failed to open file: " << filename << '\n'; - std::exit(EXIT_FAILURE); - } - - for (std::uint64_t i = 0; i < lanes; ++i) { - out << "lane[" << i << "] = " << buffer[i] << '\n'; - std::cout << "lane[" << i << "] = " << buffer[i] << '\n'; - - } - - // Important because std::exit() does not run local destructors - out.close(); - // std::exit(EXIT_SUCCESS); -} - -#include -#include -#include - -static inline void print_svint8(const char *name, svint8_t v) -{ - size_t vl = svcntb(); // Number of int8 lanes in current SVE vector - int8_t buf[vl]; - - svst1_s8(svptrue_b8(), buf, v); - - printf("%s = [", name); - for (size_t i = 0; i < vl; ++i) { - printf("%s%d", i ? ", " : "", (int)buf[i]); - } - printf("]\n"); -} - - -void print_sve_u8(const char* label, svuint8_t v) -{ - // Number of 8‑bit lanes for the current SVE implementation - const int vl = svcntb(); - - // Stack buffer (GCC/Clang extension; fine for debug) - alignas(64) uint8_t buf[svcntb()]; - - // Predicate: all lanes active - svbool_t pg = svptrue_b8(); - - // Store vector to memory - svst1(pg, buf, v); - - // Print - std::cout << label << " (vl=" << vl << "): "; - for (int i = 0; i < vl; ++i) { - std::cout << static_cast(buf[i]) << ' '; - } - std::cout << '\n'; -} - - -void print_sve_s32(const char* label, svint32_t v) -{ - // Number of 32‑bit lanes for the current SVE implementation - const int vl = svcntw(); - - // Stack buffer sized at runtime (allowed in C++, but GCC/Clang extension) - alignas(64) int32_t buf[svcntw()]; - - // Predicate: all lanes active - svbool_t pg = svptrue_b32(); - - // Store vector to memory - svst1(pg, buf, v); - - // Print - std::cout << label << " (vl=" << vl << "): "; - for (int i = 0; i < vl; ++i) { - std::cout << buf[i] << ' '; - } - std::cout << '\n'; -} - -void print_sve_f32(const char* label, svfloat32_t v) -{ - // Number of 32‑bit floating‑point lanes - const int vl = svcntw(); - - // Portable C++ buffer - std::vector buf(vl); - - // Predicate: all lanes active - svbool_t pg = svptrue_b32(); - - // Store SVE vector to memory - svst1(pg, buf.data(), v); - - // Print - std::cout << label << " (vl=" << vl << "): "; - for (float x : buf) { - std::cout << x << ' '; - } - std::cout << '\n'; -} - - #if defined(__aarch64__) && defined(__ARM_NEON) && (defined(__ARM_FEATURE_MATMUL_INT8) || defined(__ARM_FEATURE_DOTPROD)) // Helper for decoding scales and mins of Q4_K and Q5_K block formats static inline void decode_q_Kx8_6bit_scales(const uint8_t * scales_in, int16x8_t * out_mins, int8_t * out_scales) { @@ -198,31 +49,8 @@ static inline void decode_q_Kx8_6bit_scales(const uint8_t * scales_in, int16x8_t } #endif - -bool saveBytesToFile(const uint8_t* ptr, std::size_t size, const std::string& filename) -{ - std::ofstream out(filename, std::ios::binary); - if (!out) { - return false; - } - - out.write(reinterpret_cast(ptr), size); - return static_cast(out); -} - -bool loadBytesFromFile(uint8_t* ptr, std::size_t size, const std::string& filename) -{ - std::ifstream in(filename, std::ios::binary); - if (!in) { - return false; - } - - in.read(reinterpret_cast(ptr), size); - return in.gcount() == static_cast(size); -} - #if defined(__aarch64__) && defined(__ARM_FEATURE_SVE) && (defined(__ARM_FEATURE_MATMUL_INT8) || defined(__ARM_FEATURE_DOTPROD)) -// Helper for decoding scales and mins of Q4_K and Q5_K block formats +// Helper for decoding scales and mins of Q4_K and Q5_K block formats for SVE static inline void decode_q_Kx8_6bit_scales_sve(const uint8_t * scales_in, svint16_t * out_mins, int8_t * out_scales) { constexpr uint32_t kmask1 = 0x3f3f3f3f; constexpr uint32_t kmask2 = 0x0f0f0f0f; @@ -231,42 +59,15 @@ static inline void decode_q_Kx8_6bit_scales_sve(const uint8_t * scales_in, svint uint32_t sm[3]; memcpy(sm, scales_in, scales_size); - // std::cout< loadedData{}; - // uint8_t* loadPtr = loadedData.data(); - - // if (!loadBytesFromFile(loadPtr, loadedData.size(), "data.txt")) { - // std::cerr << "Failed to load file\n"; - // return; - // } - - // std::cout << "Loaded Data" << std::endl << "scales_in: "; - // for (uint8_t value : loadedData) { - // std::cout << static_cast(value) << " "; - // } - // std::cout << "\nout_mins: \n"; const uint32_t mins_0_3 = sm[1] & kmask1; - const uint32_t mins_4_7 = ((sm[2] >> 4) & kmask2) | (((sm[1] >> 6) & kmask3) << 4); - // const uint32x2_t mins_u32 = { mins_0_3, mins_4_7 }; + const uint32_t mins_4_7 = ((sm[2] >> 4) & kmask2) | (((sm[1] >> 6) & kmask3) << 4); uint32_t tmp_mins[2] = { mins_0_3, mins_4_7 }; svbool_t pg2_u32 = svptrue_pat_b32(SV_VL4); // 2 x uint32_t = 64 bits svuint32_t mins_u32 = svld1_u32(pg2_u32, tmp_mins); - *out_mins = svreinterpret_s16_u16(svunpklo_u16(svreinterpret_u8_u32(mins_u32))); - // dump_sve_s16_to_file_and_exit("out_mins_s16.txt", *out_mins); - uint32_t scales_u32[2]; scales_u32[0] = sm[0] & kmask1; scales_u32[1] = (sm[2] & kmask2) | (((sm[0] >> 6) & kmask3) << 4); - // std::cout<<"\nout_scales: "<< scales_u32[0] <<" - "< Date: Thu, 16 Jul 2026 11:12:00 +0530 Subject: [PATCH 14/19] review apoints addressed --- ggml/src/ggml-cpu/arch/arm/repack.cpp | 147 ++++++++++++++------------ 1 file changed, 80 insertions(+), 67 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cpp index c3f63775636a..9f874f45866d 100644 --- a/ggml/src/ggml-cpu/arch/arm/repack.cpp +++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp @@ -14,7 +14,6 @@ #include #include // for qsort #include // for GGML_ASSERT -#include #define GGML_CPU_CLANG_WORKAROUND #include "../../repack.h" @@ -4111,6 +4110,8 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, { const svuint8_t m4b = svdup_n_u8(0x0f); svbool_t pg_b8_vl16 = svptrue_pat_b8(SV_VL16); + const uint32_t perm_arr[4] = { 0, 2, 1, 3 }; + svuint32_t perm = svld1_u32(svptrue_pat_b32(SV_VL4), perm_arr); for (int y = 0; y < nr / q8_k_blocklen; y++) { const block_q8_Kx4 * GGML_RESTRICT q8_ptr = (const block_q8_Kx4 *) vy + (y * nb); @@ -4172,13 +4173,15 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, // 2 * 12 = 24 bytes per subblock, 4 sbs -> 4 * 24 = 96 bytes total int8_t q4sb_scales[2][8]; svint16_t q4sb_mins_0, q4sb_mins_1; // int16 as its needed for bias_acc later - for (int i = 0; i < 2; i++) { - const int offset = sb * 24 + i * 12; - if(i==0) - decode_q_Kx8_6bit_scales_sve(&q4_ptr[b].scales[offset], &q4sb_mins_0, q4sb_scales[i]); - else - decode_q_Kx8_6bit_scales_sve(&q4_ptr[b].scales[offset], &q4sb_mins_1, q4sb_scales[i]); - } + // for (int i = 0; i < 2; i++) { + // const int offset = sb * 24 + i * 12; + // if(i==0) + // decode_q_Kx8_6bit_scales_sve(&q4_ptr[b].scales[offset], &q4sb_mins_0, q4sb_scales[i]); + // else + // decode_q_Kx8_6bit_scales_sve(&q4_ptr[b].scales[offset], &q4sb_mins_1, q4sb_scales[i]); + // } + decode_q_Kx8_6bit_scales_sve(&q4_ptr[b].scales[sb * 24 + 0 * 12], &q4sb_mins_0, q4sb_scales[0]); + decode_q_Kx8_6bit_scales_sve(&q4_ptr[b].scales[sb * 24 + 1 * 12], &q4sb_mins_1, q4sb_scales[1]); // q8_ptr[b].qs has interleaved Q8 rows (01, 23) const int8_t * q8_base = q8_ptr[b].qs + sb * 256; @@ -4225,36 +4228,36 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, // Calculates the Qs muladd of every row pair (rp) rows 01 and 23 of q8 //Low nibbles of q4 and first 4 bytes of row 01 - svint32_t acc = sb_acc_0; - acc = svmmla_s32(acc, q4_nibbles_00, q8_qs_01_0); - acc = svmmla_s32(acc, q4_nibbles_01, q8_qs_01_1); - acc = svmmla_s32(acc, q4_nibbles_02, q8_qs_01_2); - acc = svmmla_s32(acc, q4_nibbles_03, q8_qs_01_3); - sb_acc_0 = acc; + // svint32_t acc = sb_acc_0; + sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_00, q8_qs_01_0); + sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_01, q8_qs_01_1); + sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_02, q8_qs_01_2); + sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_03, q8_qs_01_3); + // sb_acc_0 = acc; //High nibbles of q4 and next 4 bytes of row 01 - acc = sb_acc_1; - acc = svmmla_s32(acc, q4_nibbles_10, q8_qs_01_4); - acc = svmmla_s32(acc, q4_nibbles_11, q8_qs_01_5); - acc = svmmla_s32(acc, q4_nibbles_12, q8_qs_01_6); - acc = svmmla_s32(acc, q4_nibbles_13, q8_qs_01_7); - sb_acc_1 = acc; + // acc = sb_acc_1; + sb_acc_1 = svmmla_s32(sb_acc_1, q4_nibbles_10, q8_qs_01_4); + sb_acc_1 = svmmla_s32(sb_acc_1, q4_nibbles_11, q8_qs_01_5); + sb_acc_1 = svmmla_s32(sb_acc_1, q4_nibbles_12, q8_qs_01_6); + sb_acc_1 = svmmla_s32(sb_acc_1, q4_nibbles_13, q8_qs_01_7); + // sb_acc_1 = acc; //Low nibbles of q4 and first 4 bytes of row 23 - acc = sb_acc_2; - acc = svmmla_s32(acc, q4_nibbles_00, q8_qs_23_0); - acc = svmmla_s32(acc, q4_nibbles_01, q8_qs_23_1); - acc = svmmla_s32(acc, q4_nibbles_02, q8_qs_23_2); - acc = svmmla_s32(acc, q4_nibbles_03, q8_qs_23_3); - sb_acc_2 = acc; + // acc = sb_acc_2; + sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_00, q8_qs_23_0); + sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_01, q8_qs_23_1); + sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_02, q8_qs_23_2); + sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_03, q8_qs_23_3); + // sb_acc_2 = acc; //High nibbles of q4 and next 4 bytes of row 23 - acc = sb_acc_3; - acc = svmmla_s32(acc, q4_nibbles_10, q8_qs_23_4); - acc = svmmla_s32(acc, q4_nibbles_11, q8_qs_23_5); - acc = svmmla_s32(acc, q4_nibbles_12, q8_qs_23_6); - acc = svmmla_s32(acc, q4_nibbles_13, q8_qs_23_7); - sb_acc_3 = acc; + // acc = sb_acc_3; + sb_acc_3 = svmmla_s32(sb_acc_3, q4_nibbles_10, q8_qs_23_4); + sb_acc_3 = svmmla_s32(sb_acc_3, q4_nibbles_11, q8_qs_23_5); + sb_acc_3 = svmmla_s32(sb_acc_3, q4_nibbles_12, q8_qs_23_6); + sb_acc_3 = svmmla_s32(sb_acc_3, q4_nibbles_13, q8_qs_23_7); + // sb_acc_3 = acc; // Scales[i] corresponds to column i const int scale_offset = cp * 2; @@ -4325,9 +4328,7 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, } // for sb - // Reorder of i8mm output with bias and output layout - const uint32_t perm_arr[4] = { 0, 2, 1, 3 }; - svuint32_t perm = svld1_u32(svptrue_pat_b32(SV_VL4), perm_arr); + // Reorder of i8mm output with bias and output layout acc_00 = svtbl_s32(acc_00, perm); acc_11 = svtbl_s32(acc_11, perm); acc_22 = svtbl_s32(acc_22, perm); @@ -4430,39 +4431,51 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, // Predicate for exactly 4 lanes svbool_t pg4 = svptrue_pat_b32(SV_VL4); - for (int i = 0; i < q8_k_blocklen; i++) { - int row = y * q8_k_blocklen + i; - for (int j = 0; j < 2; j++) { - int col = x * ncols_interleaved + j * 4; - int offset = row * bs + col; + // for (int i = 0; i < q8_k_blocklen; i++) { + // int row = y * q8_k_blocklen + i; + // for (int j = 0; j < 2; j++) { + // int col = x * ncols_interleaved + j * 4; + // int offset = row * bs + col; + + // if (i == 0 && j == 0) { + // // acc_f32_0 → lower half of acc_f32_01 + // svst1_f32(pg4, s + offset, acc_f32_00); + // } else if (i == 0 && j == 1) { + // // acc_f32_1 → upper half of acc_f32_01 + // svst1_f32(pg4, s + offset, acc_f32_11); + // } else if (i == 1 && j == 0) { + // // acc_f32_2 + // svst1_f32(pg4, s + offset, acc_f32_22); + // } else if (i == 1 && j == 1) { + // // acc_f32_3 + // svst1_f32(pg4, s + offset, acc_f32_33); + // } else if (i == 2 && j == 0) { + // // acc_f32_4 + // svst1_f32(pg4, s + offset, acc_f32_44); + // } else if (i == 2 && j == 1) { + // // acc_f32_5 + // svst1_f32(pg4, s + offset, acc_f32_55); + // } else if (i == 3 && j == 0) { + // // acc_f32_6 + // svst1_f32(pg4, s + offset, acc_f32_66); + // } else if (i == 3 && j == 1) { + // // acc_f32_7 + // svst1_f32(pg4, s + offset, acc_f32_77); + // } + // } + // } + + // 8 values in total -> i from 0 to 3 and j from 0 to 1 + // (y * q8_k_blocklen + i)* bs + x * ncols_interleaved + j * 4; + svst1_f32(pg4, s + (y * q8_k_blocklen + 0)* bs + x * ncols_interleaved + 0 * 4, acc_f32_00); + svst1_f32(pg4, s + (y * q8_k_blocklen + 0)* bs + x * ncols_interleaved + 1 * 4, acc_f32_00); + svst1_f32(pg4, s + (y * q8_k_blocklen + 1)* bs + x * ncols_interleaved + 0 * 4, acc_f32_00); + svst1_f32(pg4, s + (y * q8_k_blocklen + 1)* bs + x * ncols_interleaved + 1 * 4, acc_f32_00); + svst1_f32(pg4, s + (y * q8_k_blocklen + 2)* bs + x * ncols_interleaved + 0 * 4, acc_f32_00); + svst1_f32(pg4, s + (y * q8_k_blocklen + 2)* bs + x * ncols_interleaved + 1 * 4, acc_f32_00); + svst1_f32(pg4, s + (y * q8_k_blocklen + 3)* bs + x * ncols_interleaved + 0 * 4, acc_f32_00); + svst1_f32(pg4, s + (y * q8_k_blocklen + 3)* bs + x * ncols_interleaved + 1 * 4, acc_f32_00); - if (i == 0 && j == 0) { - // acc_f32_0 → lower half of acc_f32_01 - svst1_f32(pg4, s + offset, acc_f32_00); - } else if (i == 0 && j == 1) { - // acc_f32_1 → upper half of acc_f32_01 - svst1_f32(pg4, s + offset, acc_f32_11); - } else if (i == 1 && j == 0) { - // acc_f32_2 - svst1_f32(pg4, s + offset, acc_f32_22); - } else if (i == 1 && j == 1) { - // acc_f32_3 - svst1_f32(pg4, s + offset, acc_f32_33); - } else if (i == 2 && j == 0) { - // acc_f32_4 - svst1_f32(pg4, s + offset, acc_f32_44); - } else if (i == 2 && j == 1) { - // acc_f32_5 - svst1_f32(pg4, s + offset, acc_f32_55); - } else if (i == 3 && j == 0) { - // acc_f32_6 - svst1_f32(pg4, s + offset, acc_f32_66); - } else if (i == 3 && j == 1) { - // acc_f32_7 - svst1_f32(pg4, s + offset, acc_f32_77); - } - } - } } // for x } // for y return; From 60bd01b0d0848deabf8de8ed4884dc14985869e0 Mon Sep 17 00:00:00 2001 From: Anubhav Srivastava Date: Thu, 16 Jul 2026 11:47:03 +0530 Subject: [PATCH 15/19] accuracy issue fixed --- ggml/src/ggml-cpu/arch/arm/repack.cpp | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cpp index 9f874f45866d..4a82708e197b 100644 --- a/ggml/src/ggml-cpu/arch/arm/repack.cpp +++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp @@ -3784,6 +3784,7 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, constexpr int ncols_interleaved = 8; constexpr int blocklen = 8; + constexpr int q8_k_blocklen = 4; assert(n % qk == 0); assert(nr % 4 == 0); @@ -3795,8 +3796,6 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, #if defined(__aarch64__) && defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8) - constexpr int q8_k_blocklen = 4; - switch(svcntb() * 8){ case 256: { @@ -4468,13 +4467,13 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, // 8 values in total -> i from 0 to 3 and j from 0 to 1 // (y * q8_k_blocklen + i)* bs + x * ncols_interleaved + j * 4; svst1_f32(pg4, s + (y * q8_k_blocklen + 0)* bs + x * ncols_interleaved + 0 * 4, acc_f32_00); - svst1_f32(pg4, s + (y * q8_k_blocklen + 0)* bs + x * ncols_interleaved + 1 * 4, acc_f32_00); - svst1_f32(pg4, s + (y * q8_k_blocklen + 1)* bs + x * ncols_interleaved + 0 * 4, acc_f32_00); - svst1_f32(pg4, s + (y * q8_k_blocklen + 1)* bs + x * ncols_interleaved + 1 * 4, acc_f32_00); - svst1_f32(pg4, s + (y * q8_k_blocklen + 2)* bs + x * ncols_interleaved + 0 * 4, acc_f32_00); - svst1_f32(pg4, s + (y * q8_k_blocklen + 2)* bs + x * ncols_interleaved + 1 * 4, acc_f32_00); - svst1_f32(pg4, s + (y * q8_k_blocklen + 3)* bs + x * ncols_interleaved + 0 * 4, acc_f32_00); - svst1_f32(pg4, s + (y * q8_k_blocklen + 3)* bs + x * ncols_interleaved + 1 * 4, acc_f32_00); + svst1_f32(pg4, s + (y * q8_k_blocklen + 0)* bs + x * ncols_interleaved + 1 * 4, acc_f32_11); + svst1_f32(pg4, s + (y * q8_k_blocklen + 1)* bs + x * ncols_interleaved + 0 * 4, acc_f32_22); + svst1_f32(pg4, s + (y * q8_k_blocklen + 1)* bs + x * ncols_interleaved + 1 * 4, acc_f32_33); + svst1_f32(pg4, s + (y * q8_k_blocklen + 2)* bs + x * ncols_interleaved + 0 * 4, acc_f32_44); + svst1_f32(pg4, s + (y * q8_k_blocklen + 2)* bs + x * ncols_interleaved + 1 * 4, acc_f32_55); + svst1_f32(pg4, s + (y * q8_k_blocklen + 3)* bs + x * ncols_interleaved + 0 * 4, acc_f32_66); + svst1_f32(pg4, s + (y * q8_k_blocklen + 3)* bs + x * ncols_interleaved + 1 * 4, acc_f32_77); } // for x } // for y From 9b33d1a2a66d9fbde375acafcd7bbf1399fb6f01 Mon Sep 17 00:00:00 2001 From: Anubhav Srivastava Date: Thu, 16 Jul 2026 19:23:51 +0530 Subject: [PATCH 16/19] resuing q4d_min and q4_d --- ggml/src/ggml-cpu/arch/arm/repack.cpp | 72 +++++++++++++-------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cpp index 4a82708e197b..f8f7a13a73cc 100644 --- a/ggml/src/ggml-cpu/arch/arm/repack.cpp +++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp @@ -4348,80 +4348,80 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, // i=0, j=0 svfloat32_t q8_d = svdup_n_f32(q8_ptr[b].d[0]); - svfloat32_t q4_dmin = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].dmin + 0 * 4)), svdup_f16((__fp16)0.0))); - svfloat32_t dmins = svmul_f32_x(svptrue_b32(), q4_dmin, q8_d); - svfloat32_t q4_d = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].d + 0 * 4)), svdup_f16((__fp16)0.0))); - svfloat32_t scale = svmul_f32_x(svptrue_b32(), q4_d, q8_d); + svfloat32_t q4_dmin_0 = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].dmin + 0 * 4)), svdup_f16((__fp16)0.0))); + svfloat32_t dmins = svmul_f32_x(svptrue_b32(), q4_dmin_0, q8_d); + svfloat32_t q4_d_0 = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].d + 0 * 4)), svdup_f16((__fp16)0.0))); + svfloat32_t scale = svmul_f32_x(svptrue_b32(), q4_d_0, q8_d); acc_f32_00 = svmls_f32_m(svptrue_b32(), acc_f32_00, svcvt_f32_s32_x(svptrue_b32(), bias_acc_00), dmins); acc_f32_00 = svmla_f32_m(svptrue_b32(), acc_f32_00, svcvt_f32_s32_x(svptrue_b32(), reorder_acc_0), scale); //i == 0, j == 1 - q8_d = svdup_n_f32(q8_ptr[b].d[0]); - q4_dmin = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].dmin + 1 * 4)), svdup_f16((__fp16)0.0))); - dmins = svmul_f32_x(svptrue_b32(), q4_dmin, q8_d); - q4_d = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].d + 1 * 4)), svdup_f16((__fp16)0.0))); - scale = svmul_f32_x(svptrue_b32(), q4_d, q8_d); + // q8_d = svdup_n_f32(q8_ptr[b].d[0]); + svfloat32_t q4_dmin_1 = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].dmin + 1 * 4)), svdup_f16((__fp16)0.0))); + dmins = svmul_f32_x(svptrue_b32(), q4_dmin_1, q8_d); + svfloat32_t q4_d_1 = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].d + 1 * 4)), svdup_f16((__fp16)0.0))); + scale = svmul_f32_x(svptrue_b32(), q4_d_1, q8_d); acc_f32_11 = svmls_f32_m(svptrue_b32(), acc_f32_11, svcvt_f32_s32_x(svptrue_b32(), bias_acc_11), dmins); acc_f32_11 = svmla_f32_m(svptrue_b32(), acc_f32_11, svcvt_f32_s32_x(svptrue_b32(), reorder_acc_1), scale); //i == 1, j == 0 q8_d = svdup_n_f32(q8_ptr[b].d[1]); - q4_dmin = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].dmin + 0 * 4)), svdup_f16((__fp16)0.0))); - dmins = svmul_f32_x(svptrue_b32(), q4_dmin, q8_d); - q4_d = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].d + 0 * 4)), svdup_f16((__fp16)0.0))); - scale = svmul_f32_x(svptrue_b32(), q4_d, q8_d); + // q4_dmin = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].dmin + 0 * 4)), svdup_f16((__fp16)0.0))); + dmins = svmul_f32_x(svptrue_b32(), q4_dmin_0, q8_d); + // q4_d = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].d + 0 * 4)), svdup_f16((__fp16)0.0))); + scale = svmul_f32_x(svptrue_b32(), q4_d_0, q8_d); acc_f32_22 = svmls_f32_m(svptrue_b32(), acc_f32_22, svcvt_f32_s32_x(svptrue_b32(), bias_acc_22), dmins); acc_f32_22 = svmla_f32_m(svptrue_b32(), acc_f32_22, svcvt_f32_s32_x(svptrue_b32(), reorder_acc_2), scale); //i == 1, j == 1 - q8_d = svdup_n_f32(q8_ptr[b].d[1]); - q4_dmin = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].dmin + 1 * 4)), svdup_f16((__fp16)0.0))); - dmins = svmul_f32_x(svptrue_b32(), q4_dmin, q8_d); - q4_d = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].d + 1 * 4)), svdup_f16((__fp16)0.0))); - scale = svmul_f32_x(svptrue_b32(), q4_d, q8_d); + // q8_d = svdup_n_f32(q8_ptr[b].d[1]); + // q4_dmin = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].dmin + 1 * 4)), svdup_f16((__fp16)0.0))); + dmins = svmul_f32_x(svptrue_b32(), q4_dmin_1, q8_d); + // q4_d = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].d + 1 * 4)), svdup_f16((__fp16)0.0))); + scale = svmul_f32_x(svptrue_b32(), q4_d_1, q8_d); acc_f32_33 = svmls_f32_m(svptrue_b32(), acc_f32_33, svcvt_f32_s32_x(svptrue_b32(), bias_acc_33), dmins); acc_f32_33 = svmla_f32_m(svptrue_b32(), acc_f32_33, svcvt_f32_s32_x(svptrue_b32(), reorder_acc_3), scale); //i == 2, j == 0 q8_d = svdup_n_f32(q8_ptr[b].d[2]); - q4_dmin = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].dmin + 0 * 4)), svdup_f16((__fp16)0.0))); - dmins = svmul_f32_x(svptrue_b32(), q4_dmin, q8_d); - q4_d = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].d + 0 * 4)), svdup_f16((__fp16)0.0))); - scale = svmul_f32_x(svptrue_b32(), q4_d, q8_d); + // q4_dmin = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].dmin + 0 * 4)), svdup_f16((__fp16)0.0))); + dmins = svmul_f32_x(svptrue_b32(), q4_dmin_0, q8_d); + // q4_d = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].d + 0 * 4)), svdup_f16((__fp16)0.0))); + scale = svmul_f32_x(svptrue_b32(), q4_d_0, q8_d); acc_f32_44 = svmls_f32_m(svptrue_b32(), acc_f32_44, svcvt_f32_s32_x(svptrue_b32(), bias_acc_44), dmins); acc_f32_44 = svmla_f32_m(svptrue_b32(), acc_f32_44, svcvt_f32_s32_x(svptrue_b32(), reorder_acc_4), scale); //i == 2, j == 1 - q8_d = svdup_n_f32(q8_ptr[b].d[2]); - q4_dmin = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].dmin + 1 * 4)), svdup_f16((__fp16)0.0))); - dmins = svmul_f32_x(svptrue_b32(), q4_dmin, q8_d); - q4_d = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].d + 1 * 4)), svdup_f16((__fp16)0.0))); - scale = svmul_f32_x(svptrue_b32(), q4_d, q8_d); + // q8_d = svdup_n_f32(q8_ptr[b].d[2]); + // q4_dmin = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].dmin + 1 * 4)), svdup_f16((__fp16)0.0))); + dmins = svmul_f32_x(svptrue_b32(), q4_dmin_1, q8_d); + // q4_d = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].d + 1 * 4)), svdup_f16((__fp16)0.0))); + scale = svmul_f32_x(svptrue_b32(), q4_d_1, q8_d); acc_f32_55 = svmls_f32_m(svptrue_b32(), acc_f32_55, svcvt_f32_s32_x(svptrue_b32(), bias_acc_55), dmins); acc_f32_55 = svmla_f32_m(svptrue_b32(), acc_f32_55, svcvt_f32_s32_x(svptrue_b32(), reorder_acc_5), scale); //i == 3, j == 0 q8_d = svdup_n_f32(q8_ptr[b].d[3]); - q4_dmin = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].dmin + 0 * 4)), svdup_f16((__fp16)0.0))); - dmins = svmul_f32_x(svptrue_b32(), q4_dmin, q8_d); - q4_d = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].d + 0 * 4)), svdup_f16((__fp16)0.0))); - scale = svmul_f32_x(svptrue_b32(), q4_d, q8_d); + // q4_dmin = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].dmin + 0 * 4)), svdup_f16((__fp16)0.0))); + dmins = svmul_f32_x(svptrue_b32(), q4_dmin_0, q8_d); + // q4_d = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].d + 0 * 4)), svdup_f16((__fp16)0.0))); + scale = svmul_f32_x(svptrue_b32(), q4_d_0, q8_d); acc_f32_66 = svmls_f32_m(svptrue_b32(), acc_f32_66, svcvt_f32_s32_x(svptrue_b32(), bias_acc_66), dmins); acc_f32_66 = svmla_f32_m(svptrue_b32(), acc_f32_66, svcvt_f32_s32_x(svptrue_b32(), reorder_acc_6), scale); //i == 3, j == 1 - q8_d = svdup_n_f32(q8_ptr[b].d[3]); - q4_dmin = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].dmin + 1 * 4)), svdup_f16((__fp16)0.0))); - dmins = svmul_f32_x(svptrue_b32(), q4_dmin, q8_d); - q4_d = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].d + 1 * 4)), svdup_f16((__fp16)0.0))); - scale = svmul_f32_x(svptrue_b32(), q4_d, q8_d); + // q8_d = svdup_n_f32(q8_ptr[b].d[3]); + // q4_dmin = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].dmin + 1 * 4)), svdup_f16((__fp16)0.0))); + dmins = svmul_f32_x(svptrue_b32(), q4_dmin_1, q8_d); + // q4_d = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].d + 1 * 4)), svdup_f16((__fp16)0.0))); + scale = svmul_f32_x(svptrue_b32(), q4_d_1, q8_d); acc_f32_77 = svmls_f32_m(svptrue_b32(), acc_f32_77, svcvt_f32_s32_x(svptrue_b32(), bias_acc_77), dmins); acc_f32_77 = svmla_f32_m(svptrue_b32(), acc_f32_77, svcvt_f32_s32_x(svptrue_b32(), reorder_acc_7), scale); From 0eebfae33de2ec78cdd29a067cbf00072f60d8ae Mon Sep 17 00:00:00 2001 From: Anubhav Srivastava Date: Fri, 17 Jul 2026 10:16:29 +0530 Subject: [PATCH 17/19] formatting changes SVE 256 --- ggml/src/ggml-cpu/arch/arm/repack.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cpp index f8f7a13a73cc..9cce167a8b33 100644 --- a/ggml/src/ggml-cpu/arch/arm/repack.cpp +++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp @@ -3795,13 +3795,12 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, UNUSED(blocklen); #if defined(__aarch64__) && defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8) - switch(svcntb() * 8){ case 256: { const svuint8_t m4b_1 = svdup_n_u8(0x0f); - //SV_VL8 would enable lower 8 lanes, since we're only loading 8 VALUES here this should work for higher VLs as well + // 8 accumulators: 2 row pairs × 4 col pairs svfloat32_t acc_f32_01, acc_f32_23, acc_f32_45, acc_f32_67; uint32_t idx_arr[8] = { 0, 2, 4, 6, 1, 3, 5, 7 }; svbool_t pg = svptrue_pat_b32(SV_VL8); @@ -3821,7 +3820,7 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, acc_f32_45 = svdup_n_f32(0); acc_f32_67 = svdup_n_f32(0); - for (int b = 0; b < nb; b++) { //nb is number of quantization blocks + for (int b = 0; b < nb; b++) { // bsums pairs belongs to the same q8_k subblock // 64 elements loaded and made sum of 0-7 and 8-15 sum || 16-23 and 24 - 31 sum const int16x8_t bsums[4]{ @@ -3930,7 +3929,7 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, block_scale_1 = svtbl_s32(svzip2_s32(S01_d, R01_d), idx); block_scale_2 = svtbl_s32(svzip1_s32(S23_d, R23_d), idx); block_scale_3 = svtbl_s32(svzip2_s32(S23_d, R23_d), idx); - } //This Q4_K scale computation should also remain the same + } const int8_t * q8_base_1 = q8_ptr[b].qs + sb * 256; From ce43d273df9f7d6a705e6779202b84e5edcb9614 Mon Sep 17 00:00:00 2001 From: Anubhav Srivastava Date: Fri, 17 Jul 2026 10:20:37 +0530 Subject: [PATCH 18/19] extra new line --- ggml/src/ggml-cpu/arch/arm/repack.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cpp index 9cce167a8b33..97ed53de8afc 100644 --- a/ggml/src/ggml-cpu/arch/arm/repack.cpp +++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp @@ -3799,7 +3799,6 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, case 256: { const svuint8_t m4b_1 = svdup_n_u8(0x0f); - // 8 accumulators: 2 row pairs × 4 col pairs svfloat32_t acc_f32_01, acc_f32_23, acc_f32_45, acc_f32_67; uint32_t idx_arr[8] = { 0, 2, 4, 6, 1, 3, 5, 7 }; From e117e2e4b979bee3471b65aeea88fb10f4759460 Mon Sep 17 00:00:00 2001 From: Anubhav Srivastava Date: Wed, 22 Jul 2026 09:51:30 +0530 Subject: [PATCH 19/19] cleanup --- ggml/src/ggml-cpu/arch/arm/repack.cpp | 73 ++------------------------- 1 file changed, 4 insertions(+), 69 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/arm/repack.cpp b/ggml/src/ggml-cpu/arch/arm/repack.cpp index 97ed53de8afc..2355fce8ecc4 100644 --- a/ggml/src/ggml-cpu/arch/arm/repack.cpp +++ b/ggml/src/ggml-cpu/arch/arm/repack.cpp @@ -3799,7 +3799,7 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, case 256: { const svuint8_t m4b_1 = svdup_n_u8(0x0f); - // 8 accumulators: 2 row pairs × 4 col pairs + // 4 accumulators: 2 row pairs × 4 col pairs svfloat32_t acc_f32_01, acc_f32_23, acc_f32_45, acc_f32_67; uint32_t idx_arr[8] = { 0, 2, 4, 6, 1, 3, 5, 7 }; svbool_t pg = svptrue_pat_b32(SV_VL8); @@ -4169,14 +4169,8 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, // Need scales for the low and high nibbles // 2 * 12 = 24 bytes per subblock, 4 sbs -> 4 * 24 = 96 bytes total int8_t q4sb_scales[2][8]; - svint16_t q4sb_mins_0, q4sb_mins_1; // int16 as its needed for bias_acc later - // for (int i = 0; i < 2; i++) { - // const int offset = sb * 24 + i * 12; - // if(i==0) - // decode_q_Kx8_6bit_scales_sve(&q4_ptr[b].scales[offset], &q4sb_mins_0, q4sb_scales[i]); - // else - // decode_q_Kx8_6bit_scales_sve(&q4_ptr[b].scales[offset], &q4sb_mins_1, q4sb_scales[i]); - // } + svint16_t q4sb_mins_0, q4sb_mins_1; + decode_q_Kx8_6bit_scales_sve(&q4_ptr[b].scales[sb * 24 + 0 * 12], &q4sb_mins_0, q4sb_scales[0]); decode_q_Kx8_6bit_scales_sve(&q4_ptr[b].scales[sb * 24 + 1 * 12], &q4sb_mins_1, q4sb_scales[1]); @@ -4225,36 +4219,28 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, // Calculates the Qs muladd of every row pair (rp) rows 01 and 23 of q8 //Low nibbles of q4 and first 4 bytes of row 01 - // svint32_t acc = sb_acc_0; sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_00, q8_qs_01_0); sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_01, q8_qs_01_1); sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_02, q8_qs_01_2); sb_acc_0 = svmmla_s32(sb_acc_0, q4_nibbles_03, q8_qs_01_3); - // sb_acc_0 = acc; //High nibbles of q4 and next 4 bytes of row 01 - // acc = sb_acc_1; sb_acc_1 = svmmla_s32(sb_acc_1, q4_nibbles_10, q8_qs_01_4); sb_acc_1 = svmmla_s32(sb_acc_1, q4_nibbles_11, q8_qs_01_5); sb_acc_1 = svmmla_s32(sb_acc_1, q4_nibbles_12, q8_qs_01_6); sb_acc_1 = svmmla_s32(sb_acc_1, q4_nibbles_13, q8_qs_01_7); - // sb_acc_1 = acc; //Low nibbles of q4 and first 4 bytes of row 23 - // acc = sb_acc_2; sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_00, q8_qs_23_0); sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_01, q8_qs_23_1); sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_02, q8_qs_23_2); sb_acc_2 = svmmla_s32(sb_acc_2, q4_nibbles_03, q8_qs_23_3); - // sb_acc_2 = acc; //High nibbles of q4 and next 4 bytes of row 23 - // acc = sb_acc_3; sb_acc_3 = svmmla_s32(sb_acc_3, q4_nibbles_10, q8_qs_23_4); sb_acc_3 = svmmla_s32(sb_acc_3, q4_nibbles_11, q8_qs_23_5); sb_acc_3 = svmmla_s32(sb_acc_3, q4_nibbles_12, q8_qs_23_6); sb_acc_3 = svmmla_s32(sb_acc_3, q4_nibbles_13, q8_qs_23_7); - // sb_acc_3 = acc; // Scales[i] corresponds to column i const int scale_offset = cp * 2; @@ -4322,7 +4308,6 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, bias_acc_66 = svmla_s32_x(pg_s32, bias_acc_66, svdup_n_s32((int32_t)bsums_arr[sb][7]), svunpklo_s32(q4sb_mins_1)); bias_acc_77 = svmla_s32_x(pg_s32, bias_acc_77, svdup_n_s32((int32_t)bsums_arr[sb][6]), svunpkhi_s32(q4sb_mins_0)); bias_acc_77 = svmla_s32_x(pg_s32, bias_acc_77, svdup_n_s32((int32_t)bsums_arr[sb][7]), svunpkhi_s32(q4sb_mins_1)); - } // for sb // Reorder of i8mm output with bias and output layout @@ -4355,7 +4340,6 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, acc_f32_00 = svmla_f32_m(svptrue_b32(), acc_f32_00, svcvt_f32_s32_x(svptrue_b32(), reorder_acc_0), scale); //i == 0, j == 1 - // q8_d = svdup_n_f32(q8_ptr[b].d[0]); svfloat32_t q4_dmin_1 = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].dmin + 1 * 4)), svdup_f16((__fp16)0.0))); dmins = svmul_f32_x(svptrue_b32(), q4_dmin_1, q8_d); svfloat32_t q4_d_1 = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].d + 1 * 4)), svdup_f16((__fp16)0.0))); @@ -4366,19 +4350,14 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, //i == 1, j == 0 q8_d = svdup_n_f32(q8_ptr[b].d[1]); - // q4_dmin = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].dmin + 0 * 4)), svdup_f16((__fp16)0.0))); dmins = svmul_f32_x(svptrue_b32(), q4_dmin_0, q8_d); - // q4_d = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].d + 0 * 4)), svdup_f16((__fp16)0.0))); scale = svmul_f32_x(svptrue_b32(), q4_d_0, q8_d); acc_f32_22 = svmls_f32_m(svptrue_b32(), acc_f32_22, svcvt_f32_s32_x(svptrue_b32(), bias_acc_22), dmins); acc_f32_22 = svmla_f32_m(svptrue_b32(), acc_f32_22, svcvt_f32_s32_x(svptrue_b32(), reorder_acc_2), scale); //i == 1, j == 1 - // q8_d = svdup_n_f32(q8_ptr[b].d[1]); - // q4_dmin = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].dmin + 1 * 4)), svdup_f16((__fp16)0.0))); dmins = svmul_f32_x(svptrue_b32(), q4_dmin_1, q8_d); - // q4_d = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].d + 1 * 4)), svdup_f16((__fp16)0.0))); scale = svmul_f32_x(svptrue_b32(), q4_d_1, q8_d); acc_f32_33 = svmls_f32_m(svptrue_b32(), acc_f32_33, svcvt_f32_s32_x(svptrue_b32(), bias_acc_33), dmins); @@ -4386,19 +4365,14 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, //i == 2, j == 0 q8_d = svdup_n_f32(q8_ptr[b].d[2]); - // q4_dmin = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].dmin + 0 * 4)), svdup_f16((__fp16)0.0))); dmins = svmul_f32_x(svptrue_b32(), q4_dmin_0, q8_d); - // q4_d = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].d + 0 * 4)), svdup_f16((__fp16)0.0))); scale = svmul_f32_x(svptrue_b32(), q4_d_0, q8_d); acc_f32_44 = svmls_f32_m(svptrue_b32(), acc_f32_44, svcvt_f32_s32_x(svptrue_b32(), bias_acc_44), dmins); acc_f32_44 = svmla_f32_m(svptrue_b32(), acc_f32_44, svcvt_f32_s32_x(svptrue_b32(), reorder_acc_4), scale); //i == 2, j == 1 - // q8_d = svdup_n_f32(q8_ptr[b].d[2]); - // q4_dmin = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].dmin + 1 * 4)), svdup_f16((__fp16)0.0))); dmins = svmul_f32_x(svptrue_b32(), q4_dmin_1, q8_d); - // q4_d = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].d + 1 * 4)), svdup_f16((__fp16)0.0))); scale = svmul_f32_x(svptrue_b32(), q4_d_1, q8_d); acc_f32_55 = svmls_f32_m(svptrue_b32(), acc_f32_55, svcvt_f32_s32_x(svptrue_b32(), bias_acc_55), dmins); @@ -4406,19 +4380,14 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, //i == 3, j == 0 q8_d = svdup_n_f32(q8_ptr[b].d[3]); - // q4_dmin = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].dmin + 0 * 4)), svdup_f16((__fp16)0.0))); dmins = svmul_f32_x(svptrue_b32(), q4_dmin_0, q8_d); - // q4_d = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].d + 0 * 4)), svdup_f16((__fp16)0.0))); scale = svmul_f32_x(svptrue_b32(), q4_d_0, q8_d); acc_f32_66 = svmls_f32_m(svptrue_b32(), acc_f32_66, svcvt_f32_s32_x(svptrue_b32(), bias_acc_66), dmins); acc_f32_66 = svmla_f32_m(svptrue_b32(), acc_f32_66, svcvt_f32_s32_x(svptrue_b32(), reorder_acc_6), scale); //i == 3, j == 1 - // q8_d = svdup_n_f32(q8_ptr[b].d[3]); - // q4_dmin = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].dmin + 1 * 4)), svdup_f16((__fp16)0.0))); dmins = svmul_f32_x(svptrue_b32(), q4_dmin_1, q8_d); - // q4_d = svcvt_f32_f16_z(svptrue_b32(), svzip1_f16(svld1_f16(svptrue_pat_b16(SV_VL4),(const __fp16 *) (q4_ptr[b].d + 1 * 4)), svdup_f16((__fp16)0.0))); scale = svmul_f32_x(svptrue_b32(), q4_d_1, q8_d); acc_f32_77 = svmls_f32_m(svptrue_b32(), acc_f32_77, svcvt_f32_s32_x(svptrue_b32(), bias_acc_77), dmins); @@ -4428,40 +4397,7 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, // Predicate for exactly 4 lanes svbool_t pg4 = svptrue_pat_b32(SV_VL4); - // for (int i = 0; i < q8_k_blocklen; i++) { - // int row = y * q8_k_blocklen + i; - // for (int j = 0; j < 2; j++) { - // int col = x * ncols_interleaved + j * 4; - // int offset = row * bs + col; - - // if (i == 0 && j == 0) { - // // acc_f32_0 → lower half of acc_f32_01 - // svst1_f32(pg4, s + offset, acc_f32_00); - // } else if (i == 0 && j == 1) { - // // acc_f32_1 → upper half of acc_f32_01 - // svst1_f32(pg4, s + offset, acc_f32_11); - // } else if (i == 1 && j == 0) { - // // acc_f32_2 - // svst1_f32(pg4, s + offset, acc_f32_22); - // } else if (i == 1 && j == 1) { - // // acc_f32_3 - // svst1_f32(pg4, s + offset, acc_f32_33); - // } else if (i == 2 && j == 0) { - // // acc_f32_4 - // svst1_f32(pg4, s + offset, acc_f32_44); - // } else if (i == 2 && j == 1) { - // // acc_f32_5 - // svst1_f32(pg4, s + offset, acc_f32_55); - // } else if (i == 3 && j == 0) { - // // acc_f32_6 - // svst1_f32(pg4, s + offset, acc_f32_66); - // } else if (i == 3 && j == 1) { - // // acc_f32_7 - // svst1_f32(pg4, s + offset, acc_f32_77); - // } - // } - // } - + // 8 values in total -> i from 0 to 3 and j from 0 to 1 // (y * q8_k_blocklen + i)* bs + x * ncols_interleaved + j * 4; svst1_f32(pg4, s + (y * q8_k_blocklen + 0)* bs + x * ncols_interleaved + 0 * 4, acc_f32_00); @@ -4472,7 +4408,6 @@ void ggml_gemm_q4_K_8x8_q8_K(int n, svst1_f32(pg4, s + (y * q8_k_blocklen + 2)* bs + x * ncols_interleaved + 1 * 4, acc_f32_55); svst1_f32(pg4, s + (y * q8_k_blocklen + 3)* bs + x * ncols_interleaved + 0 * 4, acc_f32_66); svst1_f32(pg4, s + (y * q8_k_blocklen + 3)* bs + x * ncols_interleaved + 1 * 4, acc_f32_77); - } // for x } // for y return;