diff --git a/.github/actions/windows-setup-cuda/action.yml b/.github/actions/windows-setup-cuda/action.yml index e67b6321e95a..fe05c29ad1c8 100644 --- a/.github/actions/windows-setup-cuda/action.yml +++ b/.github/actions/windows-setup-cuda/action.yml @@ -100,6 +100,37 @@ runs: echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 echo "CUDA_PATH_V13_1=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + - name: Install Cuda Toolkit 13.3 + if: ${{ inputs.cuda_version == '13.3' }} + shell: pwsh + run: | + mkdir -p "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" + choco install unzip -y + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_crt/windows-x86_64/cuda_crt-windows-x86_64-13.3.33-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-13.3.29-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-13.3.33-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-13.3.33-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-13.5.1.27-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.3.33-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-13.3.29-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.3.27-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.3.27-archive.zip" + curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.3.3.1-archive.zip" + unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_crt-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_cudart-windows-x86_64-13.3.29-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_nvcc-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_nvrtc-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\libcublas-windows-x86_64-13.5.1.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\libnvvm-windows-x86_64-13.3.33-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_nvtx-windows-x86_64-13.3.29-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cuda_profiler_api-windows-x86_64-13.3.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\visual_studio_integration-windows-x86_64-13.3.27-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\cccl-windows-x86_64-13.3.3.3.1-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" /E /I /H /Y + echo "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + echo "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + echo "CUDA_PATH_V13_3=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + - name: Install Cuda Toolkit 13.4 for x64 if: ${{ inputs.cuda_version == '13.4' && inputs.cuda_arch == 'x64' }} shell: pwsh diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cpp index f4d99319a133..077ff8b329dd 100644 --- a/ggml/src/ggml-sycl/mmvq.cpp +++ b/ggml/src/ggml-sycl/mmvq.cpp @@ -4840,49 +4840,6 @@ static bool dispatch_mul_mat_vec_q_glu_plain(const void * vgate, const void * vu return false; } -bool ggml_sycl_mul_mat_vec_q_glu_reorder(enum ggml_type src0_type, enum ggml_glu_op glu_op, const void * vx, - const void * vgate, const void * vy, float * dst, int ncols, int nrows, - int ncols_dst, int stride_col_y_bytes, int stride_col_dst, - dpct::queue_ptr stream) { - if (glu_op != GGML_GLU_OP_SWIGLU && glu_op != GGML_GLU_OP_GEGLU) { - return false; - } - switch (src0_type) { - case GGML_TYPE_Q4_K: - return mul_mat_vec_q_glu_reorder_type(glu_op, vx, vgate, vy, dst, ncols, nrows, ncols_dst, - stride_col_y_bytes, stride_col_dst, stream); - case GGML_TYPE_PQ2_0: - return mul_mat_vec_q_glu_reorder_type(glu_op, vx, vgate, vy, dst, ncols, nrows, ncols_dst, - stride_col_y_bytes, stride_col_dst, stream); - case GGML_TYPE_Q6_K: // ARC-LAB: GGML_SYCL_Q6K_GLU (fusion.cpp) - return mul_mat_vec_q_glu_reorder_type(glu_op, vx, vgate, vy, dst, ncols, nrows, ncols_dst, - stride_col_y_bytes, stride_col_dst, stream); - case GGML_TYPE_PTQ1_0: - if (ncols_dst == 1 && ggml_sycl_ptq1_0_glu_n1_enabled()) { - static const int glu_nr = getenv("GGML_SYCL_PTQ1_GLU1_NR") ? atoi(getenv("GGML_SYCL_PTQ1_GLU1_NR")) : 2; - if (glu_nr == 4) { - launch_ptq1_0_pairs_glu<4>(vx, vgate, vy, dst, ncols, nrows, glu_op, stream); - } else { - launch_ptq1_0_pairs_glu<2>(vx, vgate, vy, dst, ncols, nrows, glu_op, stream); - } - return true; - } - if (ptq1_0_ncols_dec_enabled() && ncols_dst >= 1 && ncols_dst <= 8) { -#define PTQ1_DEC_GLU(N) case N: launch_ptq1_0_ncols_dec(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, \ - stride_col_dst, glu_op, stream); return true; - switch (ncols_dst) { - PTQ1_DEC_GLU(1) PTQ1_DEC_GLU(2) PTQ1_DEC_GLU(3) PTQ1_DEC_GLU(4) - PTQ1_DEC_GLU(5) PTQ1_DEC_GLU(6) PTQ1_DEC_GLU(7) PTQ1_DEC_GLU(8) - } -#undef PTQ1_DEC_GLU - } - return mul_mat_vec_q_glu_reorder_type(glu_op, vx, vgate, vy, dst, ncols, nrows, ncols_dst, - stride_col_y_bytes, stride_col_dst, stream); - default: - return false; - } -} - // ARC-LAB: reordered q4_0 x f32 for 5..64 columns (speculative verify / n-gram drafts) on XMX via s8 x s4 DPAS // (inline vISA, xe2-dpas.hpp; Xe2). The MMVQ column kernels re-read the weights per 8 columns and the dequantize // + GEMM path converts the whole matrix; here each weight row streams once for all columns. One lane per weight row: B = @@ -5074,6 +5031,33 @@ bool ggml_sycl_mul_mat_vec_q_glu_reorder(enum ggml_type src0_type, enum ggml_glu // fusion declines q5_K past 5 columns return mul_mat_vec_q_glu_reorder_ncols( glu_op, vx, vgate, vy, dst, ncols, nrows, ncols_dst, stride_col_y_bytes, stride_col_dst, stream); + case GGML_TYPE_PQ2_0: + return mul_mat_vec_q_glu_reorder_type(glu_op, vx, vgate, vy, dst, ncols, nrows, ncols_dst, + stride_col_y_bytes, stride_col_dst, stream); + case GGML_TYPE_Q6_K: // ARC-LAB: GGML_SYCL_Q6K_GLU (fusion.cpp) + return mul_mat_vec_q_glu_reorder_type(glu_op, vx, vgate, vy, dst, ncols, nrows, ncols_dst, + stride_col_y_bytes, stride_col_dst, stream); + case GGML_TYPE_PTQ1_0: + if (ncols_dst == 1 && ggml_sycl_ptq1_0_glu_n1_enabled()) { + static const int glu_nr = getenv("GGML_SYCL_PTQ1_GLU1_NR") ? atoi(getenv("GGML_SYCL_PTQ1_GLU1_NR")) : 2; + if (glu_nr == 4) { + launch_ptq1_0_pairs_glu<4>(vx, vgate, vy, dst, ncols, nrows, glu_op, stream); + } else { + launch_ptq1_0_pairs_glu<2>(vx, vgate, vy, dst, ncols, nrows, glu_op, stream); + } + return true; + } + if (ptq1_0_ncols_dec_enabled() && ncols_dst >= 1 && ncols_dst <= 8) { +#define PTQ1_DEC_GLU(N) case N: launch_ptq1_0_ncols_dec(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, \ + stride_col_dst, glu_op, stream); return true; + switch (ncols_dst) { + PTQ1_DEC_GLU(1) PTQ1_DEC_GLU(2) PTQ1_DEC_GLU(3) PTQ1_DEC_GLU(4) + PTQ1_DEC_GLU(5) PTQ1_DEC_GLU(6) PTQ1_DEC_GLU(7) PTQ1_DEC_GLU(8) + } +#undef PTQ1_DEC_GLU + } + return mul_mat_vec_q_glu_reorder_type(glu_op, vx, vgate, vy, dst, ncols, nrows, ncols_dst, + stride_col_y_bytes, stride_col_dst, stream); default: return false; }