diff --git a/ggml/src/ggml-hexagon/CMakeLists.txt b/ggml/src/ggml-hexagon/CMakeLists.txt index 845b922bfedb..d6c063952507 100644 --- a/ggml/src/ggml-hexagon/CMakeLists.txt +++ b/ggml/src/ggml-hexagon/CMakeLists.txt @@ -22,92 +22,174 @@ message(STATUS "hexagon: using ${HEXAGON_SDK_ROOT} and ${HEXAGON_TOOLS_ROOT} for include(${HEXAGON_SDK_ROOT}/build/cmake/hexagon_fun.cmake) include(ExternalProject) +option(GGML_HEXAGON_USE_MEMPOOL "ggml-hexagon: mempool/FastRPC-invoke AP implementation (single mempool, synchronous invoke); OFF = dspqueue/per-buffer implementation" OFF) option(GGML_HEXAGON_HTP_DEBUG "ggml-hexagon: enable HTP debug output" OFF) -set(GGML_HEXAGON_HTP_CERT "$ENV{HEXAGON_HTP_CERT}" CACHE PATH "ggml-hexagon: enable HTP library signing using certificate") -add_library(htp_iface OBJECT - ${CMAKE_CURRENT_BINARY_DIR}/htp_iface_stub.c) +set(TARGET_NAME ggml-hexagon) + +if (GGML_HEXAGON_USE_MEMPOOL) + if (NOT CMAKE_SYSTEM_NAME MATCHES Android) + message(FATAL_ERROR "ggml-hexagon (mempool) is only available on Android") + endif() + + add_library(ggml_dsp OBJECT + ${CMAKE_CURRENT_BINARY_DIR}/ggml_dsp_stub.c) + + set_target_properties(ggml_dsp PROPERTIES POSITION_INDEPENDENT_CODE ON) + + target_include_directories(ggml_dsp PUBLIC + ${HEXAGON_SDK_ROOT}/incs + ${HEXAGON_SDK_ROOT}/incs/stddef + ${HEXAGON_SDK_ROOT}/utils/examples + ${CMAKE_CURRENT_SOURCE_DIR}/htp + ${CMAKE_CURRENT_BINARY_DIR}) -set_target_properties(htp_iface PROPERTIES POSITION_INDEPENDENT_CODE ON) -target_include_directories(htp_iface PUBLIC - ${HEXAGON_SDK_ROOT}/incs - ${HEXAGON_SDK_ROOT}/incs/stddef - ${HEXAGON_SDK_ROOT}/utils/examples - ${CMAKE_CURRENT_SOURCE_DIR}/htp - ${CMAKE_CURRENT_BINARY_DIR}) + build_idl(htp/ggml_dsp.idl ggml_dsp) -build_idl(htp/htp_iface.idl htp_iface) + if (CMAKE_SYSTEM_NAME MATCHES Android) + target_link_options(ggml_dsp PUBLIC -llog -ldl) + elseif (CMAKE_SYSTEM_NAME MATCHES Windows) + target_precompile_headers(ggml_dsp PUBLIC ) + else() + target_link_options(ggml_dsp PUBLIC -ldl) + endif() -if (CMAKE_SYSTEM_NAME MATCHES Android) - target_link_options(htp_iface PUBLIC -llog -ldl) -elseif (CMAKE_SYSTEM_NAME MATCHES Windows) - target_precompile_headers(htp_iface PUBLIC ) + ggml_add_backend_library(${TARGET_NAME} + ggml-hexagon-fastrpc.cpp + htp-drv.cpp + htp-drv.h + libdl.h + ../../include/ggml-hexagon.h) + + target_include_directories(${TARGET_NAME} PRIVATE + ${HEXAGON_SDK_ROOT}/incs + ${HEXAGON_SDK_ROOT}/incs/stddef + ${CMAKE_CURRENT_SOURCE_DIR}/htp + ${CMAKE_CURRENT_BINARY_DIR}) + + target_link_libraries(${TARGET_NAME} PRIVATE ggml_dsp) + target_include_directories(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/htp ${CMAKE_CURRENT_BINARY_DIR}) + + # Build HTP skels + set(HTP_SKELS) + function(build_htp_skel_mempool V) + ExternalProject_Add(htp-mempool-${V} + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/htp BUILD_ALWAYS ON + BUILD_BYPRODUCTS ${CMAKE_BINARY_DIR}/bin/libggml-htp-${V}.so + CMAKE_ARGS + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_TOOLCHAIN_FILE=${CMAKE_CURRENT_SOURCE_DIR}/htp/cmake-toolchain.cmake + -DCMAKE_INSTALL_LIBDIR=${CMAKE_BINARY_DIR}/bin + -DHEXAGON_SDK_ROOT=${HEXAGON_SDK_ROOT} + -DHEXAGON_TOOLS_ROOT=${HEXAGON_TOOLS_ROOT} + -DHEXAGON_HTP_DEBUG=${GGML_HEXAGON_HTP_DEBUG} + -DDSP_VERSION=${V} + -DGGML_HEXAGON_USE_MEMPOOL=ON + -DPREBUILT_LIB_DIR="toolv19_${V}") + list(APPEND HTP_SKELS ${CMAKE_CURRENT_BINARY_DIR}/libggml-htp-${V}.so) + set(HTP_SKELS ${HTP_SKELS} PARENT_SCOPE) + endfunction() + + build_htp_skel_mempool(v73) + build_htp_skel_mempool(v75) + build_htp_skel_mempool(v79) + build_htp_skel_mempool(v81) + + # Install Hexagon skels required at runtime + install(FILES ${HTP_SKELS} TYPE LIB) + + if (EXISTS ${CMAKE_SOURCE_DIR}/scripts/ggml-hexagon.cfg) + add_custom_command(TARGET ${TARGET_NAME} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + ${CMAKE_SOURCE_DIR}/scripts/ggml-hexagon.cfg ${CMAKE_BINARY_DIR}/bin/ + COMMENT "copy ggml-hexagon.cfg") + endif() else() - target_link_options(htp_iface PUBLIC -ldl) -endif() + set(GGML_HEXAGON_HTP_CERT "$ENV{HEXAGON_HTP_CERT}" CACHE PATH "ggml-hexagon: enable HTP library signing using certificate") + + add_library(htp_iface OBJECT + ${CMAKE_CURRENT_BINARY_DIR}/htp_iface_stub.c) + + set_target_properties(htp_iface PROPERTIES POSITION_INDEPENDENT_CODE ON) + target_include_directories(htp_iface PUBLIC + ${HEXAGON_SDK_ROOT}/incs + ${HEXAGON_SDK_ROOT}/incs/stddef + ${HEXAGON_SDK_ROOT}/utils/examples + ${CMAKE_CURRENT_SOURCE_DIR}/htp + ${CMAKE_CURRENT_BINARY_DIR}) + + build_idl(htp/htp_iface.idl htp_iface) + + if (CMAKE_SYSTEM_NAME MATCHES Android) + target_link_options(htp_iface PUBLIC -llog -ldl) + elseif (CMAKE_SYSTEM_NAME MATCHES Windows) + target_precompile_headers(htp_iface PUBLIC ) + else() + target_link_options(htp_iface PUBLIC -ldl) + endif() -set(TARGET_NAME ggml-hexagon) -ggml_add_backend_library(${TARGET_NAME} - ggml-hexagon.cpp - htp-drv.cpp - htp-drv.h - libdl.h - ../../include/ggml-hexagon.h) - -target_link_libraries(${TARGET_NAME} PRIVATE htp_iface) -target_include_directories(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/htp ${CMAKE_CURRENT_BINARY_DIR}) - -# Build HTP skels -set(HTP_SKELS) -function(build_htp_skel V) - ExternalProject_Add(htp-${V} - SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/htp BUILD_ALWAYS ON - BUILD_BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/libggml-htp-${V}.so - CMAKE_ARGS - -DCMAKE_BUILD_TYPE=Release - -DCMAKE_TOOLCHAIN_FILE=${CMAKE_CURRENT_SOURCE_DIR}/htp/cmake-toolchain.cmake - -DCMAKE_INSTALL_LIBDIR=${CMAKE_CURRENT_BINARY_DIR} - -DHEXAGON_SDK_ROOT=${HEXAGON_SDK_ROOT} - -DHEXAGON_TOOLS_ROOT=${HEXAGON_TOOLS_ROOT} - -DHEXAGON_HTP_DEBUG=${GGML_HEXAGON_HTP_DEBUG} - -DDSP_VERSION=${V} - -DPREBUILT_LIB_DIR="toolv19_${V}") - list(APPEND HTP_SKELS ${CMAKE_CURRENT_BINARY_DIR}/libggml-htp-${V}.so) - set(HTP_SKELS ${HTP_SKELS} PARENT_SCOPE) -endfunction() - -build_htp_skel(v73) -build_htp_skel(v75) -build_htp_skel(v79) -build_htp_skel(v81) - -# Install Hexagon skels required at runtime -install(FILES ${HTP_SKELS} TYPE LIB) - -if (CMAKE_SYSTEM_NAME MATCHES Windows AND GGML_HEXAGON_HTP_CERT) - file(TO_CMAKE_PATH "$ENV{WINDOWS_SDK_BIN}/arm64" WINSDK_BIN0_ARM64) - file(TO_CMAKE_PATH "$ENV{WINDOWS_SDK_BIN}/x86" WINSDK_BIN0_X86) - file(TO_CMAKE_PATH "$ENV{WindowsSdkVerBinPath}/arm64" WINSDK_BIN1_ARM64) - file(TO_CMAKE_PATH "$ENV{WindowsSdkVerBinPath}/x86" WINSDK_BIN1_X86) - - set(WINSDK_PATHS ${WINSDK_BIN0_ARM64} ${WINSDK_BIN0_X86} ${WINSDK_BIN1_ARM64} ${WINSDK_BIN1_X86}) - - find_program(INF2CAT NAMES inf2cat.exe PATHS ${WINSDK_PATHS} REQUIRED) - find_program(SIGNTOOL NAMES signtool.exe PATHS ${WINSDK_PATHS} REQUIRED) - - message(STATUS "hexagon: using ${GGML_HEXAGON_HTP_CERT} to sign libggml-htp skels") - - set(LIBGGML_HTP_CAT ${CMAKE_CURRENT_BINARY_DIR}/libggml-htp.cat) - add_custom_target(libggml-htp-cat - BYPRODUCTS ${LIBGGML_HTP_CAT} - DEPENDS libggml-htp.inf ${HTP_SKELS} - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/libggml-htp.inf ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${INF2CAT} /driver:${CMAKE_CURRENT_BINARY_DIR} /os:10_25H2_ARM64 - COMMAND ${SIGNTOOL} sign /fd sha256 /f ${GGML_HEXAGON_HTP_CERT} ${LIBGGML_HTP_CAT} - COMMENT "generating and signing libggml-htp.cat file" - VERBATIM - ) - - add_dependencies(${TARGET_NAME} libggml-htp-cat) - install(FILES ${LIBGGML_HTP_CAT} TYPE LIB) + ggml_add_backend_library(${TARGET_NAME} + ggml-hexagon.cpp + htp-drv.cpp + htp-drv.h + libdl.h + ../../include/ggml-hexagon.h) + + target_link_libraries(${TARGET_NAME} PRIVATE htp_iface) + target_include_directories(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/htp ${CMAKE_CURRENT_BINARY_DIR}) + + # Build HTP skels + set(HTP_SKELS) + function(build_htp_skel V) + ExternalProject_Add(htp-${V} + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/htp BUILD_ALWAYS ON + BUILD_BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/libggml-htp-${V}.so + CMAKE_ARGS + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_TOOLCHAIN_FILE=${CMAKE_CURRENT_SOURCE_DIR}/htp/cmake-toolchain.cmake + -DCMAKE_INSTALL_LIBDIR=${CMAKE_CURRENT_BINARY_DIR} + -DHEXAGON_SDK_ROOT=${HEXAGON_SDK_ROOT} + -DHEXAGON_TOOLS_ROOT=${HEXAGON_TOOLS_ROOT} + -DHEXAGON_HTP_DEBUG=${GGML_HEXAGON_HTP_DEBUG} + -DDSP_VERSION=${V} + -DPREBUILT_LIB_DIR="toolv19_${V}") + list(APPEND HTP_SKELS ${CMAKE_CURRENT_BINARY_DIR}/libggml-htp-${V}.so) + set(HTP_SKELS ${HTP_SKELS} PARENT_SCOPE) + endfunction() + + build_htp_skel(v73) + build_htp_skel(v75) + build_htp_skel(v79) + build_htp_skel(v81) + + # Install Hexagon skels required at runtime + install(FILES ${HTP_SKELS} TYPE LIB) + + if (CMAKE_SYSTEM_NAME MATCHES Windows AND GGML_HEXAGON_HTP_CERT) + file(TO_CMAKE_PATH "$ENV{WINDOWS_SDK_BIN}/arm64" WINSDK_BIN0_ARM64) + file(TO_CMAKE_PATH "$ENV{WINDOWS_SDK_BIN}/x86" WINSDK_BIN0_X86) + file(TO_CMAKE_PATH "$ENV{WindowsSdkVerBinPath}/arm64" WINSDK_BIN1_ARM64) + file(TO_CMAKE_PATH "$ENV{WindowsSdkVerBinPath}/x86" WINSDK_BIN1_X86) + + set(WINSDK_PATHS ${WINSDK_BIN0_ARM64} ${WINSDK_BIN0_X86} ${WINSDK_BIN1_ARM64} ${WINSDK_BIN1_X86}) + + find_program(INF2CAT NAMES inf2cat.exe PATHS ${WINSDK_PATHS} REQUIRED) + find_program(SIGNTOOL NAMES signtool.exe PATHS ${WINSDK_PATHS} REQUIRED) + + message(STATUS "hexagon: using ${GGML_HEXAGON_HTP_CERT} to sign libggml-htp skels") + + set(LIBGGML_HTP_CAT ${CMAKE_CURRENT_BINARY_DIR}/libggml-htp.cat) + add_custom_target(libggml-htp-cat + BYPRODUCTS ${LIBGGML_HTP_CAT} + DEPENDS libggml-htp.inf ${HTP_SKELS} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/libggml-htp.inf ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${INF2CAT} /driver:${CMAKE_CURRENT_BINARY_DIR} /os:10_25H2_ARM64 + COMMAND ${SIGNTOOL} sign /fd sha256 /f ${GGML_HEXAGON_HTP_CERT} ${LIBGGML_HTP_CAT} + COMMENT "generating and signing libggml-htp.cat file" + VERBATIM + ) + + add_dependencies(${TARGET_NAME} libggml-htp-cat) + install(FILES ${LIBGGML_HTP_CAT} TYPE LIB) + endif() endif() diff --git a/ggml/src/ggml-hexagon/ggml-hexagon-fastrpc.cpp b/ggml/src/ggml-hexagon/ggml-hexagon-fastrpc.cpp new file mode 100644 index 000000000000..3efce7b363a4 --- /dev/null +++ b/ggml/src/ggml-hexagon/ggml-hexagon-fastrpc.cpp @@ -0,0 +1,7000 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__ANDROID__) || defined(__linux__) +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +#if defined(__ANDROID__) +#include "android/log.h" +#endif + +#include "rpcmem.h" +#include "remote.h" +#include "AEEStdErr.h" +#include "htp-drv.h" +#include "HAP_power.h" +#include "HAP_farf.h" + +#include "ggml-hexagon.h" +#include "ggml-impl.h" +#include "ggml-backend-impl.h" + +#pragma clang diagnostic ignored "-Wnested-anon-types" +#pragma clang diagnostic ignored "-Wgnu-anonymous-struct" +#define GGML_COMMON_DECL_C +#include "ggml-common.h" +#include "ggml-quants.h" + +#include "ggml_dsp.h" +#include "htp/dsp-ctx.h" +#include "htp/htp-ops.h" +#include "htp/hex-common.h" +#include "htp/hex-fastdiv.h" +#include "htp/matmul-ops.h" +#include "htp/flash-attn-ops.h" +#include "htp/unary-ops.h" + +// ================================================================================================= +// section-1: forward declarations, global vars, macros +// ================================================================================================= +#ifndef PROJECT_NAME +#define PROJECT_NAME "ggml-hexagon" +#endif + +#define GGML_HEXAGON_MAX_DEVICES 16 +#define GGML_HEXAGON_BACKEND_NAME "hexagon" + +#define GGML_DSP_IDL_VERSION "0.0.2" + +#define SIZE_IN_MB (1 << 20) + +#define GGMLHEXAGON_MAX_OPS_PER_TASK 16 + +#define GGMLHEXAGON_MAX_TENSORS_PER_TASK 32 + +#define DMA_BUF_IOCTL_SYNC_IOCTL 0x40086200u + +#if !defined (_WIN32) +#pragma weak remote_system_request +#pragma weak remote_session_control +#pragma weak remote_handle_control +#pragma weak remote_handle64_control +#pragma weak fastrpc_mmap +#pragma weak fastrpc_munmap +#endif + +// Forward declarations +struct ggml_backend_hexagon_context; +struct ggml_backend_hexagon_reg_context; + +static bool ggmlhexagon_is_metadata_op(enum ggml_op op); +static int ggmlhexagon_probe_dspinfo(ggml_backend_hexagon_context * ctx); +static const char * ggmlhexagon_get_htparch_desc(size_t htp_arch); +static size_t ggmlhexagon_get_system_total_memory_in_bytes(void); +static int ggmlhexagon_get_hvx_arch_ver(int domain, uint32_t * capability); +static int hexagon_warmup_invoke_timed(ggml_backend_hexagon_context * ctx); +static bool ggml_backend_buffer_is_hexagon_repack(const ggml_backend_buffer * b); +static bool ggml_backend_hexagon_buffer_is_host(ggml_backend_buffer_type_t buft); +static void ggmlhexagon_set_runtime_path(size_t device, const std::string & path); +static const char * ggml_backend_hexagon_buffer_type_name(ggml_backend_buffer_type_t buft); +static ggml_backend_t ggml_backend_hexagon_init_ext(size_t device, const char * runtime_libpath); +static size_t ggml_backend_hexagon_buffer_type_get_max_size(ggml_backend_buffer_type_t buft); +static size_t ggml_backend_hexagon_buffer_type_get_alignment(ggml_backend_buffer_type_t buft); +static bool ggml_backend_hexagon_repack_buffer_is_host(ggml_backend_buffer_type_t buft); +static bool ggmlhexagon_op_buffers_belong_to_dev(ggml_backend_dev_t dev, const ggml_tensor * op); +static bool ggmlhexagon_tensor_buffer_is_owned_by(ggml_backend_dev_t dev, const ggml_tensor * t); +static ggml_backend_t ggml_backend_hexagon_device_init_backend(ggml_backend_dev_t dev, const char * params); +static size_t ggml_backend_hexagon_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor); + +static ggml_backend_hexagon_context * ggml_backend_hexagon_ensure_context(ggml_backend_dev_t dev); +static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size); +static bool ggml_hexagon_compute_fa_params(const ggml_backend_hexagon_context * ctx, const ggml_tensor * node, htp_fa_kernel_params * kparams); + +GGML_BACKEND_API void ggml_hexagon_set_runtime_libpath(const char * path); + +// ================================================================================================= +// section-2: data structures +// ================================================================================================= +enum qcom_htp_arch { + NONE = 0, + V73 = 73, + V75 = 75, + V79 = 79, + V81 = 81, +}; + +enum qcom_chipset_soc_model { + UNKNOWN_SM = 0, + SM8550 = 43, // v73, SD 8 Gen 2 + SM8650 = 57, // v75, SD 8 Gen 3 + SM8750 = 69, // v79, SD 8 Elite(aka 8 Gen 4) + SM8850 = 73, // v81, SD 8 Elite Gen 5(aka 8 Gen 5) +}; + +struct qcom_socinfo { + uint32_t soc_model; + size_t htp_arch; + size_t vtcm_size_in_mb; + char soc_desc[GGML_MAX_NAME]; +}; + +// mempool region tracking. Two users: +// - backend buffers: allocated from the bump tail or reused best-fit; +// free_buffer marks the region !in_use and coalesces neighbors. +// - batch temporaries (mirrors, batch descriptors): pushed per +// graph_compute_batch call, erased on cleanup when the bump pointer +// is rolled back. +struct ion_pool_region { + size_t offset; // byte offset from mempool base + size_t size; // allocation size in bytes + bool in_use; // true if currently allocated +}; + +struct ggml_backend_hexagon_context { + int device; + char name[GGML_MAX_NAME]; + char desc[GGML_MAX_NAME]; + char lib[GGML_MAX_NAME]; + + struct ggml_backend * backend; + struct qcom_socinfo socinfo; + + int n_threads; + int dsp_thread_counts = 0; // actual worker threads in effect on DSP (max_hw_threads - 2) + + //Hexagon resource management for the general approach through Hexagon CDSP + int domain_id; + int session_id; + remote_handle64 ggmlop_handle; + size_t rpc_mempool_capacity; + size_t rpc_mempool_len; + size_t rpc_mempool_usage; + int rpc_mempool_handle; + void * rpc_mempool; + void * rpc_mempool_dsp_base; // DSP-side VA from fastrpc_mmap() (NOT from FastRPC pointer translation) + bool weights_dirty; // set by set_tensor/memset_tensor, cleared by Phase 9 + bool dsp_need_weight_inval_reset; // set by free_buffer on model unload, sent to DSP via 0xFFFC before next batch + std::vector ion_regions; // region tracking for mempool free-space management + + // FastRPC call statistics + uint64_t rpc_batch_call_count; // total ggml_dsp_execute_batch calls + int64_t cumulative_graph_us; // cumulative graph inference duration + int64_t last_graph_end_us; // wall clock of last graph end (to measure gap) + + // Session-global set of tensor data pointers that were ever a dst of any op + // in any cgraph. Used in Phase 2 to identify read-only weights. + std::unordered_set ever_dst_ptrs; + + // Per-graph node statistics + uint32_t max_nodes_per_graph; // max node count in a single graph + uint32_t min_nodes_per_graph; // min node count in a single graph + uint32_t total_nodes_processed; // cumulative node count across all graphs + + // Per-call execution time range + int64_t min_graph_us; // shortest single graph execution + int64_t max_graph_us; // longest single graph execution + uint32_t max_graph_n_nodes; // cgraph node count when max_graph_us recorded + uint32_t max_graph_n_ops; // DSP op count (post-fusion) when max_graph_us recorded + uint32_t min_n_ops_per_call; // min hex_ops.size() across all graph_compute calls + uint32_t max_n_ops_per_call; // max hex_ops.size() across all graph_compute calls + int64_t min_p10_us; // shortest single FastRPC call + int64_t max_p10_us; // longest single FastRPC call + uint32_t max_layer_idx_seen; // largest layer suffix in tensor names (ffn_gate-N etc), n_layer = max + 1 + + // Per-call AP-side overhead (graph_dur - p10). Tracks how much time each + // graph_compute_batch call spends outside of pure DSP execution + int64_t min_rpc_overhead_us; + int64_t max_rpc_overhead_us; + int64_t sum_rpc_overhead_us; + + // AP-side per-phase cumulative time + int64_t cum_p1_us; // Phase 1: collect unique tensor objects + int64_t cum_p2_us; // Phase 2: build op descriptors + int64_t cum_p3_us; // Phase 3: op fusion + int64_t cum_p4_us; // Phase 4: compute layout sizes + int64_t cum_p5_us; // Phase 5: tensor mirroring + int64_t cum_p6_us; // Phase 6: track repacked weight mempool offsets + int64_t cum_p7_us; // Phase 7: allocate batch descriptor in mempool + int64_t cum_p8_us; // Phase 8: descriptor construction + int64_t cum_p9_us; // Phase 9: cache flush + + // Phase 10: FastRPC doorbell call (cumulative) + 3-way breakdown + int64_t cum_p10_us; // cumulative FastRPC time + // p10 3-way breakdown: split the FastRPC + DSP exec + cache inval window + // so we can tell AP-side cache-coherency cost apart from DSP-side work + int64_t cum_p10_rpc_setup_us; // AP setup before ggml_dsp_execute_batch (ioctl / marshalling) + int64_t cum_p10_dsp_exec_us; // pure DSP execution time inside the sync call + int64_t cum_p10_civac_us; // AP cache invalidate after DSP reply + + int64_t cum_p11_us; // Phase 11: cache inval + int64_t cum_p12_us; // Phase 12: mempool->heap copy-back + int64_t cum_unaccounted_us; // wall-clock not covered by p1..p12 (gaps, scheduler, etc.) + + // FastRPC transport overhead calibration (measured via 0xFFFB warmup invokes at init) + // The 0xFFFB warmup mode does no DSP work, so measured time is an upper bound of + // pure FastRPC transport overhead (invoke round-trip: AP -> DSP -> AP) + int64_t rpc_overhead_min_us; // shortest warmup invoke + int64_t rpc_overhead_max_us; // longest warmup invoke + int64_t rpc_overhead_sum_us; // sum of all warmup invokes (for avg) + uint32_t rpc_overhead_count; // number of warmup invokes measured + + // Cumulative MUL_MAT counters (PP optimization diagnostics) + // Tracked in ctx so we can read rates after a sweep run without + // changing existing per-call LOG_DEBUG output paths + uint64_t n_mul_mat_total_cum = 0; // total MUL_MAT ops in supported_nodes + uint64_t n_hmx_used_cum = 0; // MUL_MAT dispatched to HMX kernels + uint64_t n_fused_qkv_cum = 0; // 3x MUL_MAT -> HTP_OP_MUL_MAT_QKV fusions + uint64_t n_fused_ffn_cum = 0; // 2x MUL_MAT -> HTP_OP_MUL_MAT_FFN fusions + uint64_t n_fused_mm_add_cum = 0; // MUL_MAT + ADD -> HTP_OP_MUL_MAT_ADD fusions + + // HMX eligibility diagnostic counters (why MUL_MATs fall back to HVX) + uint64_t n_hmx_basic_pass = 0; // passed basic HMX eligibility + uint64_t n_hmx_basic_fail_ne01 = 0; // ne01_padded %% 32 != 0 + uint64_t n_hmx_basic_fail_ne00 = 0; // ne00 %% 32 != 0 + uint64_t n_hmx_basic_fail_wtype = 0; // weight type not HMX-compatible + uint64_t n_hmx_basic_fail_batched = 0; // batched non-F16 + uint64_t n_hmx_basic_fail_permuted = 0; // nb[0] > nb[1] (permuted) + uint64_t n_hmx_basic_fail_small_n = 0; // ne11 <= HTP_MM_HMX_MIN_NROWS + uint64_t n_hmx_vtcm_pass = 0; // HMX VTCM precompute succeeded + uint64_t n_hmx_vtcm_fail = 0; // HMX VTCM precompute failed + + // Buffer type owned by this context (each device has its own buft) + struct ggml_backend_buffer_type buffer_type; + // Repack buffer type(is_host=false), same mempool as buffer_type + struct ggml_backend_buffer_type repack_buffer_type; + char buft_name[GGML_MAX_NAME]; // "hexagon-ion-buffer-", unique per device + char repack_buft_name[GGML_MAX_NAME]; // "hexagon-ion-buffer--REPACK" + + // Per-device hardware caps (probed at init, used by supports_op) + bool has_vtcm; // domain has VTCM pages available + bool has_hvx; // domain has HVX support + bool has_hmx; // domain has HMX support + bool has_async_fastrpc; // domain supports async FastRPC + bool has_extended_map; // domain supports extended (>=4 GiB) VA mapping + + // Cached htp_mm_kernel_params per (weight_data, ne11). For TG, the + // precompute math produces identical results for every token, so we + // cache the params struct to skip the multi-hundred-microsecond + // thread/chunk search on subsequent calls. + std::unordered_map mm_params_cache; + + // cgraph cache: Phase 1 (tensor dedup) + Phase 2 (hex_ops build) + + // Phase 3 (op fusion) result keyed by content-based cgraph hash. + // The scheduler rebuilds split->graph every call, so the cgraph pointer + // is NOT hashed. The underlying node ops/shapes/src/data ptrs are stable + // for graph-reuse, which is what the hash covers. + // A FNV-1a hash over {op, ne[4], nb[4], non-null src[0..GGML_MAX_SRC-1] ptr, + // data ptr, op_params} per node gives a 64-bit key that is effectively + // collision-free. + struct cgraph_cache_entry { + uint64_t content_hash = 0; + uint64_t insert_seq = 0; // FIFO stamp used by bounded-cache eviction + int n_nodes = 0; + int n_tensors = 0; + int n_ops = 0; + std::vector tensor_src; + std::vector supported_nodes; + std::vector hex_ops; + std::vector is_weight; // per-tensor boolean + }; + std::unordered_map cgraph_cache; + uint64_t cgraph_cache_hits = 0; + uint64_t cgraph_cache_misses = 0; + uint64_t cgraph_cache_seq = 0; // monotonically increasing insert stamp + + static constexpr size_t CGRAPH_CACHE_MAX = 1024; // bound distinct cached graphs + + // scratch space for Phase 9/11 cache ranges, reused to avoid per-token heap allocation in hot-path cache maintenance + std::vector> scratch_cache_ranges; + + // Phase 6: track mempool offsets for repacked quantized weights (replaces + // function-static g_tiled_ion_offsets / s_warned_non_repack that leaked across model reloads) + std::unordered_map tiled_ion_offsets; + std::unordered_set warned_non_repack; + + // QKV fusion: one-shot warning state moved from function-static to ctx member + bool warned_qkv_name; + + uint64_t set_tensor_call_count; + + ggml_backend_hexagon_context(int dev_id, ggml_backend_dev_t dev); + ~ggml_backend_hexagon_context(); +}; + +struct hexagon_appcfg_t { + int dump_debug_info; // enable/disable dump debug info for troubleshooting issues on AP side + int thread_counts; // thread_counts on CDSP side + int dump_diag_info; // enable/disable dump diag info for troubleshooting issues on CDSP side + int ndev; // number of Hexagon devices (PDs), from GGML_HEXAGON_NDEV env + int ion_sync_mode; // 0=both(DC CVAC+ion_sync), 1=ion_sync only(default), 2=DC CVAC only + int rpc_mmap_mode; // 0=FASTRPC_MAP_FD_DELAYED (default), 1=FASTRPC_MAP_FD (eager pinning) + int enable_opfusion; // 1=enable QKV/FFN op fusion (default), 0=disable (for debugging) + int fa_select; // flash attention: 2=HMX->HVX->CPU, 1=HVX->CPU, 0=CPU (default 2) + int dsp_cache_mode; // DSP-side entry.c cache optimization bitmask, pushed to DSP at init via + // execute_batch(0xFFFC) special mode. All four bits + // are wired into ggml_dsp_execute_batch() + // bit 0 (0x1): first-touch weight bitmap + // bit 1 (0x2): skip dcinva for prior dst + // bit 2 (0x4): bulk dst flush at batch end + // bit 3 (0x8): selective bulk flush - skip batch-end flush for + // dsts still consumed by a later op in the same batch (pure + // intermediates). Requires bit 2. Mirrored dsts (flags&0x1) + // and final outputs always flush. + int dsp_cache_trace_bit0; // DSP-side bit 0 (first-touch weight) trace enable + // 0=off (production) + // 1=emit one [DSP-CACHE-TRACE-BIT0] log line per bit 0 decision + // (SKIP or INVAL) with op/src/ptr/len. Pushed to DSP at init via + // bit 16 of the same execute_batch(0xFFFC) payload as dsp_cache_mode. + // Used for diagnosing the bit 0 stale-L2-read bug. + int dsp_cache_trace_bit1; // DSP-side bit 1 (skip dcinva for prior dst) trace enable + // 0=off (production) + // 1=emit one [DSP-CACHE-TRACE-BIT1] log line per bit 1 decision + // (SKIP if prior_dst_contains_src, INVAL otherwise) with + // op/src/ptr/len. Pushed to DSP at init via bit 17 of the same + // execute_batch(0xFFFC) payload. Used for diagnosing why dsp_cache_mode + // 5/6/7 garble on the new matmul pipeline (81ff7abe5). Pair with + // dsp_cache_trace_bit0 to localize the stale-L2-read culprit. + int enable_graph_optimize; // enable/disable cgraph reorder pass + + const char * cfgfilename; + const char * runtime_libpath; + char version[GGMLHEXAGON_TMPBUF_LEN]; + std::string enabled_ops; // comma-separated list of ops to offload (empty = all supported ops) + std::string enabled_types; // comma-separated list of weight types to offload for MUL_MAT (empty = all supported types) +}; + +// designated initializers are a C++20 extension; suppress to keep readability in C++17 builds +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wc++20-designator" +#endif +static struct hexagon_appcfg_t g_hexagon_appcfg = { + .dump_debug_info = 0, + .thread_counts = 6, + .dump_diag_info = 0, + .ndev = 1, + .ion_sync_mode = 1, + .rpc_mmap_mode = 0, + .enable_opfusion = 1, + .fa_select = 2, + .dsp_cache_mode = 5, + .dsp_cache_trace_bit0 = 0, + .dsp_cache_trace_bit1 = 0, + .enable_graph_optimize = 1, + .cfgfilename = "ggml-hexagon.cfg", +#if defined(__ANDROID__) + .runtime_libpath = "/data/local/tmp/", +#elif defined(__linux__) + .runtime_libpath = "/tmp/", +#elif defined(_WIN32) + .runtime_libpath = "C:\\temp\\", +#endif + .version = {"0.99.7.5"}, + .enabled_ops = "", + .enabled_types = "", +}; + +//supported Snapdragon devices with Hexagon DSP +static struct qcom_socinfo g_hexagon_soc_info_table[] = { + /* Qualcomm Snapdragon 8 Gen 2 */ + { + .soc_model = SM8550, + .htp_arch = V73, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm Snapdragon 8 Gen 2"}, + + /* Qualcomm Snapdragon 8 Gen 3 */ + { + .soc_model = SM8650, + .htp_arch = V75, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm Snapdragon 8 Gen 3 "}, + + /* Qualcomm Snapdragon 8 Gen 4 */ + { + .soc_model = SM8750, + .htp_arch = V79, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm Snapdragon 8 Elite"}, + + /* Qualcomm Snapdragon 8 Gen 5 */ + { + .soc_model = SM8850, + .htp_arch = V81, + .vtcm_size_in_mb = 8, + .soc_desc = "Qualcomm Snapdragon 8 Elite Gen5"}, +}; +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop +#endif + +// Owning pointer to the reg context. The framework's ~ggml_backend_registry() +// does not delete reg->context (see FIXME in ggml-backend-reg.cpp), so we rely +// on an atexit handler to release DSP sessions. atexit runs before static +// dtors, so function-local std::mutex objects (e.g. the log mutex) are still +// alive when ~ggml_backend_hexagon_context calls ggmlhexagon_deinit_cdsp. +static ggml_backend_hexagon_reg_context * g_reg_ctx = nullptr; + +// Static buffer for runtime libpath (updated via ggml_hexagon_set_runtime_libpath API) +static char g_runtime_libpath_buf[512] = +#if defined(__ANDROID__) + "/data/local/tmp/" +#elif defined(__linux__) + "/tmp/" +#elif defined(_WIN32) + "C:\\temp\\" +#endif +; + +// ================================================================================================= +// section-3: troubleshooting and profiler +// ================================================================================================= +static void ggmlhexagon_get_timestring(char * p_currenttime) { + if (nullptr == p_currenttime) + return; + + auto time_to_string = [](const std::chrono::system_clock::time_point & tp)->std::string { + auto as_time_t = std::chrono::system_clock::to_time_t(tp); + struct tm tm; + + localtime_r(&as_time_t, &tm); + + std::chrono::milliseconds ms = std::chrono::duration_cast(tp.time_since_epoch()); + char buf[GGMLHEXAGON_TMPBUF_LEN]; + memset(buf, 0, GGMLHEXAGON_TMPBUF_LEN); + snprintf(buf, sizeof(buf), "%04d-%02d-%02d,%02d:%02d:%02d", + tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + GGML_UNUSED(ms); + return buf; + }; + + std::chrono::system_clock::time_point tp = std::chrono::system_clock::now(); + snprintf(p_currenttime, GGMLHEXAGON_TMPBUF_LEN, "%s", time_to_string(tp).c_str()); +} + +void ggmlhexagon_log_internal(int level, const char * file, const char * func, int line, const char * format, ...) { + static std::mutex ggmlhexagon_log_internal_mutex; + static char s_ggmlhexagon_log_internal_buf[GGMLHEXAGON_LOGBUF_LEN]; + + GGML_UNUSED(file); + GGML_UNUSED(level); + + if (0 == g_hexagon_appcfg.dump_debug_info) { + return; + } + + { + std::lock_guard lock(ggmlhexagon_log_internal_mutex); + va_list args; + va_start(args, format); + int len_prefix = snprintf(s_ggmlhexagon_log_internal_buf, GGMLHEXAGON_LOGBUF_LEN, "[%s, %d]: ", func, line); + if (len_prefix < 0 || (size_t)len_prefix >= GGMLHEXAGON_LOGBUF_LEN) { + va_end(args); + return; + } + int len = vsnprintf(s_ggmlhexagon_log_internal_buf + len_prefix, GGMLHEXAGON_LOGBUF_LEN - len_prefix, format, args); + if (len >= 0 && len < (GGMLHEXAGON_LOGBUF_LEN - len_prefix)) { +#if (defined __ANDROID__) || (defined ANDROID) + __android_log_print(ANDROID_LOG_INFO, PROJECT_NAME, "%s\n", s_ggmlhexagon_log_internal_buf); + if (GGML_LOG_LEVEL_INFO == level || GGML_LOG_LEVEL_CONT == level) { + printf("%s\n", s_ggmlhexagon_log_internal_buf); + } +#else + //for Snapdragon based WoA(Windows on ARM) device or Linux + printf("%s\n", s_ggmlhexagon_log_internal_buf); +#endif + } + va_end(args); + } +} + +// Always-emit log channel. Bypasses dump_debug_info. The level parameter decides +// whether the message is also printed to stdout on Android: +// - GGML_LOG_LEVEL_NONE (ALWAYS) -> adb logcat only +// - GGML_LOG_LEVEL_ERROR -> adb logcat + terminal +// - GGML_LOG_LEVEL_CONT (VERBOSE) -> adb logcat + terminal +void ggmlhexagon_log_always_internal(int level, const char * file, const char * func, int line, const char * format, ...) { + static std::mutex s_log_mutex; + static char s_log_buf[GGMLHEXAGON_LOGBUF_LEN]; + + GGML_UNUSED(file); + GGML_UNUSED(level); + + { + std::lock_guard lock(s_log_mutex); + va_list args; + va_start(args, format); + int len_prefix = snprintf(s_log_buf, GGMLHEXAGON_LOGBUF_LEN, "[%s, %d]: ", func, line); + if (len_prefix < 0 || (size_t)len_prefix >= GGMLHEXAGON_LOGBUF_LEN) { + va_end(args); + return; + } + int len = vsnprintf(s_log_buf + len_prefix, GGMLHEXAGON_LOGBUF_LEN - len_prefix, format, args); + if (len >= 0 && len < (GGMLHEXAGON_LOGBUF_LEN - len_prefix)) { +#if (defined __ANDROID__) || (defined ANDROID) + __android_log_print(ANDROID_LOG_INFO, PROJECT_NAME, "%s\n", s_log_buf); + if (GGML_LOG_LEVEL_ERROR == level || GGML_LOG_LEVEL_CONT == level) { + printf("%s\n", s_log_buf); + } +#else + //for Snapdragon based WoA(Windows on ARM) device or Linux + printf("%s\n", s_log_buf); +#endif + } + va_end(args); + } +} + +// Invoke one no-op warmup call and record FastRPC transport overhead timing. +// The 0xFFFB warmup mode does no DSP work, so measured time is an upper bound +// of pure FastRPC transport overhead. Used at init only, no per-graph overhead. +static int hexagon_warmup_invoke_timed(ggml_backend_hexagon_context * ctx) { + int64_t t0 = ggml_time_us(); + // Use the existing no-op warmup mode (0xFFFB) instead of the legacy + // batch_size==0 probe mode so no marker bytes are written to the pool. + int err = ggml_dsp_execute_batch(ctx->ggmlop_handle, 0, 0xFFFB); + int64_t dt = ggml_time_us() - t0; + ctx->rpc_overhead_sum_us += dt; + ctx->rpc_overhead_count++; + if (ctx->rpc_overhead_min_us == 0 || dt < ctx->rpc_overhead_min_us) ctx->rpc_overhead_min_us = dt; + if (dt > ctx->rpc_overhead_max_us) ctx->rpc_overhead_max_us = dt; + return err; +} + +static void ggmlhexagon_print_running_timestamp(ggml_backend_hexagon_context * ctx) { + char timestamp[GGMLHEXAGON_TMPBUF_LEN]; + memset(timestamp, 0, GGMLHEXAGON_TMPBUF_LEN); + + GGMLHEXAGON_LOG_VERBOSE("ggml_hexagon_version: %s", g_hexagon_appcfg.version); + ggmlhexagon_get_timestring(timestamp); + GGMLHEXAGON_LOG_VERBOSE("enabled_ops: %s", "ALL"); + GGMLHEXAGON_LOG_VERBOSE("offload MUL_MAT types: %s", g_hexagon_appcfg.enabled_types.empty() ? "ALL" : g_hexagon_appcfg.enabled_types.c_str()); + GGMLHEXAGON_LOG_VERBOSE("thread_counts on CDSP: %d", g_hexagon_appcfg.thread_counts); + GGMLHEXAGON_LOG_VERBOSE("ion_sync_mode: %d", g_hexagon_appcfg.ion_sync_mode); + GGMLHEXAGON_LOG_VERBOSE("rpc_mmap_mode: %d", g_hexagon_appcfg.rpc_mmap_mode); + GGMLHEXAGON_LOG_VERBOSE("dsp_cache_mode: %d", g_hexagon_appcfg.dsp_cache_mode); + GGMLHEXAGON_LOG_VERBOSE("dsp_cache_trace_bit0: %d", g_hexagon_appcfg.dsp_cache_trace_bit0); + GGMLHEXAGON_LOG_VERBOSE("dsp_cache_trace_bit1: %d", g_hexagon_appcfg.dsp_cache_trace_bit1); + GGMLHEXAGON_LOG_VERBOSE("dump diag info(DSP): %d", g_hexagon_appcfg.dump_diag_info); + GGMLHEXAGON_LOG_VERBOSE("dump diag info(AP): %d", g_hexagon_appcfg.dump_debug_info); + GGMLHEXAGON_LOG_VERBOSE("enable graph_optimize: %d", g_hexagon_appcfg.enable_graph_optimize); + GGMLHEXAGON_LOG_VERBOSE("enable op_fusion: %d", g_hexagon_appcfg.enable_opfusion); + GGMLHEXAGON_LOG_VERBOSE("running timestamp:%s", timestamp); +} + +// dump accumulated performance statistics collected during graph_compute_batch +static void ggmlhexagon_dump_perf_stats(const ggml_backend_hexagon_context * ctx) { + if (nullptr == ctx) { + return; + } + // Logging convention in this function (inverted from the usual level names): + // VERBOSE -> terminal + adb logcat (key summary for immediate visibility) + // ALWAYS -> adb logcat only (detailed diagnostics for post-analysis) + uint32_t dsp_version = 0; + ggmlhexagon_get_hvx_arch_ver(ctx->domain_id, &dsp_version); + size_t total_mem = ggmlhexagon_get_system_total_memory_in_bytes(); + GGMLHEXAGON_LOG_VERBOSE("device info: %s, dsp arch version 0x%x, system mem size %d MiB", + ctx->socinfo.soc_desc, dsp_version, total_mem / SIZE_IN_MB); + GGMLHEXAGON_LOG_VERBOSE("device=%d name=%s arch=%s vtcm=%zuMB hvx=%d hmx=%d async_fastrpc=%d extended_map=%d", + ctx->device, ctx->name, + ggmlhexagon_get_htparch_desc(ctx->socinfo.htp_arch), + ctx->socinfo.vtcm_size_in_mb, + (int)ctx->has_hvx, (int)ctx->has_hmx, + (int)ctx->has_async_fastrpc, (int)ctx->has_extended_map); + GGMLHEXAGON_LOG_VERBOSE("model: n_layer=%u (parsed from tensor name suffixes)", + ctx->max_layer_idx_seen + 1); + GGMLHEXAGON_LOG_VERBOSE("rpc stats: batch_calls=%llu cum_p10=%lld us cum_graph=%lld us avg_p10=%lld us avg_graph=%lld us", + (unsigned long long)ctx->rpc_batch_call_count, + (long long)ctx->cum_p10_us, (long long)ctx->cumulative_graph_us, + ctx->rpc_batch_call_count ? (long long)(ctx->cum_p10_us / (int64_t)ctx->rpc_batch_call_count) : 0, + ctx->rpc_batch_call_count ? (long long)(ctx->cumulative_graph_us / (int64_t)ctx->rpc_batch_call_count) : 0); + GGMLHEXAGON_LOG_VERBOSE("graph nodes: min=%u max=%u total=%u", + ctx->min_nodes_per_graph, ctx->max_nodes_per_graph, ctx->total_nodes_processed); + GGMLHEXAGON_LOG_VERBOSE("graph ops (post-fusion): min=%u max=%u", + ctx->min_n_ops_per_call, ctx->max_n_ops_per_call); + GGMLHEXAGON_LOG_VERBOSE("per-call range: graph=[%lld, %lld] us p10=[%lld, %lld] us", + (long long)ctx->min_graph_us, (long long)ctx->max_graph_us, + (long long)ctx->min_p10_us, (long long)ctx->max_p10_us); + GGMLHEXAGON_LOG_VERBOSE("per-call overhead: n=%llu min=%lld max=%lld avg=%lld us (graph_dur - p10)", + (unsigned long long)ctx->rpc_batch_call_count, + (long long)ctx->min_rpc_overhead_us, + (long long)ctx->max_rpc_overhead_us, + ctx->rpc_batch_call_count ? (long long)(ctx->sum_rpc_overhead_us / (int64_t)ctx->rpc_batch_call_count) : 0); + GGMLHEXAGON_LOG_VERBOSE("max graph detail: dur=%lld us n_nodes=%u n_ops=%u", + (long long)ctx->max_graph_us, ctx->max_graph_n_nodes, ctx->max_graph_n_ops); + GGMLHEXAGON_LOG_VERBOSE("AP phase cumulative: p1=%lld p2=%lld p3=%lld p4=%lld p5=%lld p6=%lld p7=%lld p8=%lld p9=%lld p11=%lld p12=%lld unaccounted=%lld us", + (long long)ctx->cum_p1_us, (long long)ctx->cum_p2_us, + (long long)ctx->cum_p3_us, (long long)ctx->cum_p4_us, + (long long)ctx->cum_p5_us, (long long)ctx->cum_p6_us, + (long long)ctx->cum_p7_us, + (long long)ctx->cum_p8_us, (long long)ctx->cum_p9_us, + (long long)ctx->cum_p11_us, (long long)ctx->cum_p12_us, + (long long)ctx->cum_unaccounted_us); + // Fine-grained: 3-way p10 split + per-call distribution + GGMLHEXAGON_LOG_VERBOSE("p10 3-way cumulative: rpc_setup=%lld dsp_exec=%lld civac=%lld us (sum=%lld)", + (long long)ctx->cum_p10_rpc_setup_us, + (long long)ctx->cum_p10_dsp_exec_us, + (long long)ctx->cum_p10_civac_us, + (long long)(ctx->cum_p10_rpc_setup_us + ctx->cum_p10_dsp_exec_us + ctx->cum_p10_civac_us)); + GGMLHEXAGON_LOG_VERBOSE("rpc overhead (warmup): n=%u min=%lld max=%lld avg=%lld us (upper bound, pure FastRPC/mempool transport overhead)", + ctx->rpc_overhead_count, + (long long)ctx->rpc_overhead_min_us, (long long)ctx->rpc_overhead_max_us, + ctx->rpc_overhead_count ? (long long)(ctx->rpc_overhead_sum_us / (int64_t)ctx->rpc_overhead_count) : 0); + const uint64_t total_cache_lookups = ctx->cgraph_cache_hits + ctx->cgraph_cache_misses; + GGMLHEXAGON_LOG_VERBOSE("cgraph cache: hits=%llu misses=%llu (hit_rate=%.1f%%)", + (unsigned long long)ctx->cgraph_cache_hits, + (unsigned long long)ctx->cgraph_cache_misses, + total_cache_lookups ? (100.0 * ctx->cgraph_cache_hits / total_cache_lookups) : 0.0); + + // MUL_MAT optimization diagnostics (PP). Cumulative across the run. + // - n_mul_mat_total: every MUL_MAT in supported_nodes (cache miss only) + // - n_hmx_used: MUL_MAT dispatched to HMX (kparams.n_hmx == 1) + // - n_fused_qkv: 3x MUL_MAT (Q,K,V) merged into HTP_OP_MUL_MAT_QKV + // - n_fused_ffn: 2x MUL_MAT (gate,up) merged into HTP_OP_MUL_MAT_FFN + // - n_fused_mm_add: MUL_MAT + ADD merged into HTP_OP_MUL_MAT_ADD + { + const double total = (double) ctx->n_mul_mat_total_cum; + const double hmx_pct = total > 0 ? 100.0 * ctx->n_hmx_used_cum / total : 0.0; + const double qkv_pct = total > 0 ? 100.0 * (3 * ctx->n_fused_qkv_cum) / total : 0.0; + const double ffn_pct = total > 0 ? 100.0 * (2 * ctx->n_fused_ffn_cum) / total : 0.0; + const double add_pct = total > 0 ? 100.0 * ctx->n_fused_mm_add_cum / total : 0.0; + GGMLHEXAGON_LOG_VERBOSE("mul_mat coverage: total=%llu hmx=%llu (%.1f%%) qkv_fused=%llu (saves %.1f%%) " + "ffn_fused=%llu (saves %.1f%%) mm_add_fused=%llu (saves %.1f%%)", + (unsigned long long)ctx->n_mul_mat_total_cum, + (unsigned long long)ctx->n_hmx_used_cum, hmx_pct, + (unsigned long long)ctx->n_fused_qkv_cum, qkv_pct, + (unsigned long long)ctx->n_fused_ffn_cum, ffn_pct, + (unsigned long long)ctx->n_fused_mm_add_cum, add_pct); + } + + // HMX eligibility diagnostic: why MUL_MATs fall back to HVX + { + uint64_t basic_total = ctx->n_hmx_basic_pass + + ctx->n_hmx_basic_fail_ne01 + + ctx->n_hmx_basic_fail_ne00 + + ctx->n_hmx_basic_fail_wtype + + ctx->n_hmx_basic_fail_batched + + ctx->n_hmx_basic_fail_permuted + + ctx->n_hmx_basic_fail_small_n; + if (basic_total > 0) { + GGMLHEXAGON_LOG_VERBOSE("hmx eligibility: total=%llu pass=%llu (%.1f%%)", + (unsigned long long)basic_total, + (unsigned long long)ctx->n_hmx_basic_pass, + basic_total > 0 ? 100.0 * ctx->n_hmx_basic_pass / basic_total : 0.0); + GGMLHEXAGON_LOG_VERBOSE("hmx basic fail: ne01_align=%llu ne00_align=%llu wtype=%llu batched=%llu permuted=%llu small_n=%llu", + (unsigned long long)ctx->n_hmx_basic_fail_ne01, + (unsigned long long)ctx->n_hmx_basic_fail_ne00, + (unsigned long long)ctx->n_hmx_basic_fail_wtype, + (unsigned long long)ctx->n_hmx_basic_fail_batched, + (unsigned long long)ctx->n_hmx_basic_fail_permuted, + (unsigned long long)ctx->n_hmx_basic_fail_small_n); + GGMLHEXAGON_LOG_VERBOSE("hmx vtcm: pass=%llu fail=%llu (%.1f%% of basic-pass)", + (unsigned long long)ctx->n_hmx_vtcm_pass, + (unsigned long long)ctx->n_hmx_vtcm_fail, + ctx->n_hmx_basic_pass > 0 + ? 100.0 * ctx->n_hmx_vtcm_fail / ctx->n_hmx_basic_pass + : 0.0); + } + } +} + +// ================================================================================================= +// section-4: configuration class and helper functions +// ================================================================================================= +//a simple class to load running configurations in ggml-hexagon.cfg +class hexagon_appcfg { +public: + hexagon_appcfg() {} + + void dump(std::function worker) { + if (!_load_success) { + GGMLHEXAGON_LOG_WARN("hexagon cfg file %s not loaded", _cfg_filename.c_str()); + return; + } + auto iter = _hexagon_appcfg.begin(); + while (iter != _hexagon_appcfg.end()) { + auto kv_iter = iter->second.begin(); + while (kv_iter != iter->second.end()) { + worker(iter->first, kv_iter->first, kv_iter->second); + ++kv_iter; + } + ++iter; + } + } + + bool load(const std::string & file_name) { + if (file_name == "") { + return false; + } + _cfg_filename = file_name; + std::ifstream in; + std::string line; + in.open(file_name.c_str()); + if (!in.is_open()) { + GGMLHEXAGON_LOG_WARN("can't open file %s", file_name.c_str()); + return false; + } + while (getline(in, line)) { + std::string section, key, value; + if (!parse_line(line, section, key, value)) { + continue; + } + set_section_keyvalue(section, key, value); + } + _load_success = true; + return true; + } + + void get_stringvalue(const std::string & section, const std::string & key, std::string & value, std::string default_value) { + value = default_value; + if (_hexagon_appcfg.find(section) == _hexagon_appcfg.end()) { + return; + } + if (_hexagon_appcfg[section].find(key) == _hexagon_appcfg[section].end()) { + return; + } + value = _hexagon_appcfg[section][key]; + } + + void get_intvalue(const std::string & section, const std::string & key, int & value, int default_value) { + value = default_value; + if (_hexagon_appcfg.find(section) == _hexagon_appcfg.end()) { + return; + } + if (_hexagon_appcfg[section].find(key) == _hexagon_appcfg[section].end()) { + return; + } + value = atol(_hexagon_appcfg[section][key].c_str()); + } + +private: + void ltrim(std::string & str) { + if (str.empty()) return; + size_t len = 0; + const char * temp = str.c_str(); + while (*temp && isblank(*temp)) { + ++len; + ++temp; + } + if (len > 0) str.erase(0, len); + } + + void rtrim(std::string & str) { + if (str.empty()) return; + size_t len = str.length(); + size_t pos = len; + while (pos > 0) { + if (!isblank(str[pos - 1])) { + break; + } + --pos; + } + if (pos != len) str.erase(pos); + } + + void trim(std::string & str) { + ltrim(str); + rtrim(str); + } + + void set_section_keyvalue(std::string & section, std::string & key, std::string & value) { + if (_hexagon_appcfg.find(section) == _hexagon_appcfg.end()) { + std::unordered_map kv_map; + _hexagon_appcfg[section] = kv_map; + } + if (key != "" && value != "") _hexagon_appcfg[section][key] = value; + } + + bool parse_line(std::string & line, std::string & section, std::string & key, std::string & value) { + static std::string cur_section = ""; + std::string nodes[2] = {"#", ";"}; + for (int i = 0; i < 2; ++i) { + std::string::size_type pos = line.find(nodes[i]); + if (pos != std::string::npos) line.erase(pos); + } + trim(line); + if (line == "") return false; + if (line[0] == '[' && line[line.size() - 1] == ']') { + section = line.substr(1, line.size() - 2); + trim(section); + cur_section = section; + return false; + } + if (cur_section == "") return false; + bool is_key = true; + for (size_t i = 0; i < line.size(); ++i) { + if (line[i] == '=') { + is_key = false; + continue; + } + if (is_key) { + key += line[i]; + } else { + value += line[i]; + } + } + section = cur_section; + trim(key); + trim(value); + + //"1.00" -> 1.00 + if (value.front() == '"' && value.back() == '"') { + value.erase(0, 1); // erase the first character " + value.erase(value.size() - 1); // erase the last character " + } + + return true; + } + +private: + hexagon_appcfg(const hexagon_appcfg & ) = delete; + hexagon_appcfg(const hexagon_appcfg && ) = delete; + hexagon_appcfg & operator= (const hexagon_appcfg & ) = delete; + +private: + std::unordered_map> _hexagon_appcfg; + bool _load_success = false; + std::string _cfg_filename; +}; + +static void ggmlhexagon_load_cfg() { + //this function can be called in various scenarios + static bool initialized = false; + if (initialized) { + GGMLHEXAGON_LOG_DEBUG("hexagon appcfg file already loaded\n"); + return; + } + char time_string[GGMLHEXAGON_TMPBUF_LEN]; + memset(time_string, 0, GGMLHEXAGON_TMPBUF_LEN); + ggmlhexagon_get_timestring(time_string); + GGMLHEXAGON_LOG_DEBUG("program running start time:%s", time_string); + std::string cfg_filename = std::string(g_hexagon_appcfg.runtime_libpath) + std::string(g_hexagon_appcfg.cfgfilename); + + // Prefer user-editable copy in /data/local/tmp/ (accessible via adb shell), + // fall back to runtime_libpath/ggml-hexagon.cfg (such as app data dir in Android APK). +#if defined(__ANDROID__) + { + std::string user_cfg = "/data/local/tmp/" + std::string(g_hexagon_appcfg.cfgfilename); + if (access(user_cfg.c_str(), F_OK) == 0) { + cfg_filename = user_cfg; + } + } +#endif + + hexagon_appcfg hexagoncfg_instance; + bool cfg_loaded = hexagoncfg_instance.load(cfg_filename); + if (!cfg_loaded) { + GGMLHEXAGON_LOG_ALWAYS("cfg file %s not found or unreadable, using built-in defaults", cfg_filename.c_str()); + } + hexagoncfg_instance.dump([](const std::string & section, const std::string & key, const std::string value) { + std::ostringstream tmposs; + tmposs << "section[" << std::setw(10) << std::left << section << "],[" << std::setw(25) << std::left << key << "] = [" << value << "]"; + GGMLHEXAGON_LOG_INFO("%s", tmposs.str().c_str()); + }); + std::string version; //version of ggml-hexagon + hexagoncfg_instance.get_stringvalue("general", "version", version, "0.99.7.5"); + hexagoncfg_instance.get_intvalue("general", "dump_debug_info", g_hexagon_appcfg.dump_debug_info, 0); + + hexagoncfg_instance.get_intvalue("cdsp", "thread_counts", g_hexagon_appcfg.thread_counts, 6); + hexagoncfg_instance.get_intvalue("cdsp", "dump_diag_info", g_hexagon_appcfg.dump_diag_info, 0); + hexagoncfg_instance.get_intvalue("cdsp", "ndev", g_hexagon_appcfg.ndev, 1); + hexagoncfg_instance.get_intvalue("cdsp", "ion_sync_mode", g_hexagon_appcfg.ion_sync_mode, 1); + hexagoncfg_instance.get_intvalue("cdsp", "rpc_mmap_mode", g_hexagon_appcfg.rpc_mmap_mode, 0); + hexagoncfg_instance.get_intvalue("cdsp", "enable_opfusion", g_hexagon_appcfg.enable_opfusion, 1); + hexagoncfg_instance.get_intvalue("cdsp", "fa_select", g_hexagon_appcfg.fa_select, 2); + hexagoncfg_instance.get_intvalue("cdsp", "dsp_cache_mode", g_hexagon_appcfg.dsp_cache_mode, 5); + hexagoncfg_instance.get_intvalue("cdsp", "dsp_cache_trace_bit0", g_hexagon_appcfg.dsp_cache_trace_bit0, 0); + hexagoncfg_instance.get_intvalue("cdsp", "dsp_cache_trace_bit1", g_hexagon_appcfg.dsp_cache_trace_bit1, 0); + hexagoncfg_instance.get_intvalue("cdsp", "enable_graph_optimize", g_hexagon_appcfg.enable_graph_optimize, 1); + hexagoncfg_instance.get_stringvalue("cdsp", "enabled_ops", g_hexagon_appcfg.enabled_ops, ""); + hexagoncfg_instance.get_stringvalue("cdsp", "enabled_types", g_hexagon_appcfg.enabled_types, ""); + + snprintf(g_hexagon_appcfg.version, GGMLHEXAGON_TMPBUF_LEN, "%s", version.c_str()); + + if (cfg_loaded) { + GGMLHEXAGON_LOG_ALWAYS("load hexagon appcfg from %s", cfg_filename.c_str()); + } else { + GGMLHEXAGON_LOG_ALWAYS("no hexagon appcfg file, using built-in defaults"); + } + GGMLHEXAGON_LOG_ALWAYS("ggml_hexagon_version=%s", g_hexagon_appcfg.version); + GGMLHEXAGON_LOG_ALWAYS("runtime libpath=%s", g_hexagon_appcfg.runtime_libpath); + + // env var GGML_HEXAGON_NDEV overrides cfg value (for automation/testing) + const char * str_ndev = getenv("GGML_HEXAGON_NDEV"); + if (str_ndev) { + int v = atoi(str_ndev); + if (v > 0 && v <= GGML_HEXAGON_MAX_DEVICES) { + g_hexagon_appcfg.ndev = v; + } else { + GGMLHEXAGON_LOG_WARN("invalid GGML_HEXAGON_NDEV=%d, must be 1..%d, using cfg value %d", + v, GGML_HEXAGON_MAX_DEVICES, g_hexagon_appcfg.ndev); + } + } + if (g_hexagon_appcfg.ndev < 1 || g_hexagon_appcfg.ndev > GGML_HEXAGON_MAX_DEVICES) { + GGMLHEXAGON_LOG_WARN("invalid ndev=%d from cfg, must be 1..%d, using default 1", + g_hexagon_appcfg.ndev, GGML_HEXAGON_MAX_DEVICES); + g_hexagon_appcfg.ndev = 1; + } + GGMLHEXAGON_LOG_ALWAYS("ndev=%d (from cfg, env GGML_HEXAGON_NDEV overrides if set)", g_hexagon_appcfg.ndev); + + ggmlhexagon_set_runtime_path(0, g_hexagon_appcfg.runtime_libpath); + + initialized = true; +} + +static void ggmlhexagon_check_valid_appcfg() { + if (g_hexagon_appcfg.thread_counts < 1 || g_hexagon_appcfg.thread_counts > 6) { + GGMLHEXAGON_LOG_WARN("invalid thread_counts %d, reset to 6", g_hexagon_appcfg.thread_counts); + g_hexagon_appcfg.thread_counts = 6; + } + + if (g_hexagon_appcfg.dump_diag_info < 0 || g_hexagon_appcfg.dump_diag_info > 1) { + GGMLHEXAGON_LOG_WARN("invalid dump_diag_info %d, reset to 0", g_hexagon_appcfg.dump_diag_info); + g_hexagon_appcfg.dump_diag_info = 0; + } + + if (g_hexagon_appcfg.dsp_cache_mode < 0 || g_hexagon_appcfg.dsp_cache_mode > 15) { + GGMLHEXAGON_LOG_WARN("invalid dsp_cache_mode %d, reset to 5", g_hexagon_appcfg.dsp_cache_mode); + g_hexagon_appcfg.dsp_cache_mode = 5; + } + + if (g_hexagon_appcfg.ion_sync_mode < 0 || g_hexagon_appcfg.ion_sync_mode > 2) { + GGMLHEXAGON_LOG_WARN("invalid ion_sync_mode %d, reset to 1", g_hexagon_appcfg.ion_sync_mode); + g_hexagon_appcfg.ion_sync_mode = 1; + } + + if (g_hexagon_appcfg.rpc_mmap_mode < 0 || g_hexagon_appcfg.rpc_mmap_mode > 1) { + GGMLHEXAGON_LOG_WARN("invalid rpc_mmap_mode %d, reset to 0", g_hexagon_appcfg.rpc_mmap_mode); + g_hexagon_appcfg.rpc_mmap_mode = 0; + } + + if (g_hexagon_appcfg.fa_select < 0 || g_hexagon_appcfg.fa_select > 2) { + GGMLHEXAGON_LOG_WARN("invalid fa_select %d, reset to 2", g_hexagon_appcfg.fa_select); + g_hexagon_appcfg.fa_select = 2; + } +} + +// Set the runtime library path where DSP skeleton .so files (libggml-htp-v*.so) +// and ggml-hexagon.cfg are located. Must be called before any hexagon backend +// registration to take effect. +GGML_BACKEND_API void ggml_hexagon_set_runtime_libpath(const char * path) { + if (path == nullptr) { + return; + } + strncpy(g_runtime_libpath_buf, path, sizeof(g_runtime_libpath_buf) - 1); + g_runtime_libpath_buf[sizeof(g_runtime_libpath_buf) - 1] = '\0'; + g_hexagon_appcfg.runtime_libpath = g_runtime_libpath_buf; +} + +// ================================================================================================= +// section-5: general helper functions +// ================================================================================================= +static int ion_sync_for_direction(int fd, int direction) { +#if defined(__ANDROID__) || defined(__linux__) + if (fd <= 0) return -1; + // Flag definitions from Linux kernel include/uapi/linux/dma-buf.h: + // DMA_BUF_SYNC_READ = (1 << 0) = 1 + // DMA_BUF_SYNC_WRITE = (2 << 0) = 2 + // DMA_BUF_SYNC_START = (1 << 2) = 4 + // DMA_BUF_SYNC_END = (2 << 2) = 8 + { + const uint64_t DMA_BUF_SYNC_READ = (1u << 0); + const uint64_t DMA_BUF_SYNC_WRITE = (2u << 0); + const uint64_t DMA_BUF_SYNC_START = (1u << 2); + const uint64_t DMA_BUF_SYNC_END = (2u << 2); + uint64_t rw = (direction == 1) ? DMA_BUF_SYNC_WRITE : DMA_BUF_SYNC_READ; + struct { uint64_t flags; } s; + s.flags = DMA_BUF_SYNC_START | rw; + int r = ioctl(fd, DMA_BUF_IOCTL_SYNC_IOCTL, &s); + if (r == 0) { + s.flags = DMA_BUF_SYNC_END | rw; + r = ioctl(fd, DMA_BUF_IOCTL_SYNC_IOCTL, &s); + if (r == 0) return 0; + } + } + { + struct ion_sync_data { int fd; unsigned int flags; unsigned int pad; }; + struct ion_sync_data sync = { fd, (unsigned int)direction, 0 }; + int r = ioctl(fd, _IOWR('I', 7, struct ion_sync_data), &sync); + if (r == 0) { + return 0; + } + } +#elif defined(_WIN32) + // Cache sync is handled by dspqueue's platform-specific driver implementation. + GGML_UNUSED(fd); GGML_UNUSED(direction); +#endif + return -1; +} + +static inline void cpu_dcache_flush_range(ggml_backend_hexagon_context * backend_ctx, int ion_fd, const void * p, size_t size) { +#if defined(__ANDROID__) || defined(__linux__) + // range-based DC CVAC with 8x loop unrolling + if (size == 0) return; + { + const size_t line_size = 64; + const char * start = (const char *)((uintptr_t)p & ~(line_size - 1)); + const char * end = (const char *)p + size; + // 8x unrolled: 8 cache lines per iteration + for (; start + line_size * 8 <= end; start += line_size * 8) { + __asm__ volatile("dc cvac, %0" : : "r"((const void *)(start + line_size * 0)) : "memory"); + __asm__ volatile("dc cvac, %0" : : "r"((const void *)(start + line_size * 1)) : "memory"); + __asm__ volatile("dc cvac, %0" : : "r"((const void *)(start + line_size * 2)) : "memory"); + __asm__ volatile("dc cvac, %0" : : "r"((const void *)(start + line_size * 3)) : "memory"); + __asm__ volatile("dc cvac, %0" : : "r"((const void *)(start + line_size * 4)) : "memory"); + __asm__ volatile("dc cvac, %0" : : "r"((const void *)(start + line_size * 5)) : "memory"); + __asm__ volatile("dc cvac, %0" : : "r"((const void *)(start + line_size * 6)) : "memory"); + __asm__ volatile("dc cvac, %0" : : "r"((const void *)(start + line_size * 7)) : "memory"); + } + // tail: remaining lines + for (; start < end; start += line_size) { + __asm__ volatile("dc cvac, %0" : : "r"((const void *)start) : "memory"); + } + __asm__ volatile("dsb ish" ::: "memory"); + } + if (ion_fd > 0) ion_sync_for_direction(ion_fd, 1); +#elif defined(_WIN32) + // Cache sync is handled by dspqueue's platform-specific driver implementation. + GGML_UNUSED(backend_ctx); GGML_UNUSED(ion_fd); GGML_UNUSED(p); GGML_UNUSED(size); +#endif +} + +static inline void cpu_dcache_inval_range(ggml_backend_hexagon_context * backend_ctx, int ion_fd, const void * p, size_t size) { +#if defined(__ANDROID__) || defined(__linux__) + // range-based DC CIVAC with 8x loop unrolling + if (size == 0) return; + { + const size_t line_size = 64; + const char * start = (const char *)((uintptr_t)p & ~(line_size - 1)); + const char * end = (const char *)p + size; + // 8x unrolled: 8 cache lines per iteration + for (; start + line_size * 8 <= end; start += line_size * 8) { + __asm__ volatile("dc civac, %0" : : "r"((const void *)(start + line_size * 0)) : "memory"); + __asm__ volatile("dc civac, %0" : : "r"((const void *)(start + line_size * 1)) : "memory"); + __asm__ volatile("dc civac, %0" : : "r"((const void *)(start + line_size * 2)) : "memory"); + __asm__ volatile("dc civac, %0" : : "r"((const void *)(start + line_size * 3)) : "memory"); + __asm__ volatile("dc civac, %0" : : "r"((const void *)(start + line_size * 4)) : "memory"); + __asm__ volatile("dc civac, %0" : : "r"((const void *)(start + line_size * 5)) : "memory"); + __asm__ volatile("dc civac, %0" : : "r"((const void *)(start + line_size * 6)) : "memory"); + __asm__ volatile("dc civac, %0" : : "r"((const void *)(start + line_size * 7)) : "memory"); + } + // tail: remaining lines + for (; start < end; start += line_size) { + __asm__ volatile("dc civac, %0" : : "r"((const void *)start) : "memory"); + } + __asm__ volatile("dsb ish" ::: "memory"); + __asm__ volatile("isb" ::: "memory"); + } + if (ion_fd > 0) ion_sync_for_direction(ion_fd, 0); +#elif defined(_WIN32) + // Cache sync is handled by dspqueue's platform-specific driver implementation. + GGML_UNUSED(backend_ctx); GGML_UNUSED(ion_fd); GGML_UNUSED(p); GGML_UNUSED(size); +#endif +} + +// True for metadata-only ops that never execute on CDSP. +// Tests iterate every tensor in the graph and call supports_op on each; +// view/reshape/permute parents must be reported as supported. +static bool ggmlhexagon_is_metadata_op(enum ggml_op op) { + switch (op) { + case GGML_OP_NONE: + case GGML_OP_VIEW: + case GGML_OP_RESHAPE: + case GGML_OP_PERMUTE: + case GGML_OP_TRANSPOSE: + return true; + default: + return false; + } +} + +static const char * ggmlhexagon_get_socmodel_desc(uint32_t soc_model) { + switch (soc_model) { + case SM8550: + return "SM8550"; + case SM8650: + return "SM8650"; + case SM8750: + return "SM8750"; + case SM8850: + return "SM8850"; + default: + return "unknown"; + } +} + +//0x73 -> 73, 0x75 -> 75, 0x79 -> 79, 0x81 -> 81 +static size_t ggmlhexagon_htparch_hex_to_decimal(size_t htp_arch) { + //naive algorithm + int a = htp_arch / 16; + int b = htp_arch % 16; + return a * 10 + b; +} + +static const char * ggmlhexagon_get_htparch_desc(size_t htp_arch) { + switch (htp_arch) { + case V73: + return "QCOM_HTP_V73"; + case V75: + return "QCOM_HTP_V75"; + case V79: + return "QCOM_HTP_V79"; + case V81: + return "QCOM_HTP_V81"; + default: + return "unknown"; + } +} + +static struct qcom_socinfo * ggmlhexagon_get_socinfo_from_socmodel(uint32_t soc_model) { + size_t items = sizeof(g_hexagon_soc_info_table) / sizeof(g_hexagon_soc_info_table[0]); + for (size_t idx = 0; idx < items; idx++) { + if (soc_model == g_hexagon_soc_info_table[idx].soc_model) { + return &g_hexagon_soc_info_table[idx]; + } + } + return nullptr; +} + +static struct qcom_socinfo * ggmlhexagon_get_socinfo_from_htparch(size_t htp_arch) { + size_t items = sizeof(g_hexagon_soc_info_table) / sizeof(g_hexagon_soc_info_table[0]); + for (size_t idx = 0; idx < items; idx++) { + if (htp_arch == g_hexagon_soc_info_table[idx].htp_arch) { + return &g_hexagon_soc_info_table[idx]; + } + } + return nullptr; +} + +static size_t ggmlhexagon_get_system_total_memory_in_bytes() { +#if defined(__ANDROID__) || defined(__linux__) + struct sysinfo info = {}; + if (0 == sysinfo(&info)) { + return (info.totalram + info.totalswap) * info.mem_unit; + } + size_t pages = (size_t)sysconf(_SC_PHYS_PAGES); + size_t page_size = (size_t)sysconf(_SC_PAGE_SIZE); + + return pages * page_size; +#elif defined(_WIN32) + //TODO + return 0; +#endif +} + +static size_t ggmlhexagon_get_system_free_memory_in_bytes() { +#if defined(__ANDROID__) || defined(__linux__) + struct sysinfo info = {}; + if (0 == sysinfo(&info)) { + return (info.freeram + info.freeswap) * info.mem_unit; + } + size_t avail_pages = (size_t)sysconf(_SC_AVPHYS_PAGES); + size_t page_size = (size_t)sysconf(_SC_PAGE_SIZE); + + return avail_pages * page_size; +#elif defined(_WIN32) + //TODO + return 0; +#endif +} + +static bool ggmlhexagon_same_types(const ggml_backend_hexagon_context * ctx, const ggml_tensor * op_tensor) { + GGML_UNUSED(ctx); + ggml_tensor * src0 = op_tensor->src[0]; + ggml_tensor * src1 = op_tensor->src[1]; + if (nullptr != src1) { + if (src0->type != op_tensor->type || src1->type != op_tensor->type) { + return false; + } + } else { + if (src0->type != op_tensor->type) { + return false; + } + } + + if (src0->type != GGML_TYPE_F32) + return false; + + return true; +} + +static void ggmlhexagon_set_runtime_path(size_t device, const std::string & path) { + GGML_UNUSED(device); +#if defined(__ANDROID__) + // Android: LD_LIBRARY_PATH uses ':' as separator + std::string lib_runtime_path = path + ":/vendor/dsp/cdsp:/vendor/lib64:/vendor/dsp/dsp:/vendor/dsp/images"; + if (0 == setenv("LD_LIBRARY_PATH", lib_runtime_path.c_str(), 1)) { + GGMLHEXAGON_LOG_DEBUG("setenv LD_LIBRARY_PATH %s successfully", lib_runtime_path.c_str()); + } else { + GGMLHEXAGON_LOG_ERROR("setenv LD_LIBRARY_PATH %s failure", lib_runtime_path.c_str()); + } + + // ADSP_LIBRARY_PATH uses ';' as separator on all platforms + std::string adsp_runtime_path = path + ";/vendor/dsp/cdsp;/vendor/lib/rfsa/adsp;/system/lib/rfsa/adsp;/vendor/dsp/dsp;/vendor/dsp/images;/dsp"; + if (0 == setenv("ADSP_LIBRARY_PATH", adsp_runtime_path.c_str(), 1)) { + GGMLHEXAGON_LOG_DEBUG("setenv ADSP_LIBRARY_PATH %s successfully", adsp_runtime_path.c_str()); + } else { + GGMLHEXAGON_LOG_ERROR("setenv ADSP_LIBRARY_PATH %s failure", adsp_runtime_path.c_str()); + } +#elif defined(__linux__) + // Linux: LD_LIBRARY_PATH uses ':' as separator + std::string lib_runtime_path = path + ":/usr/local/lib:/usr/lib"; + if (0 == setenv("LD_LIBRARY_PATH", lib_runtime_path.c_str(), 1)) { + GGMLHEXAGON_LOG_DEBUG("setenv LD_LIBRARY_PATH %s successfully", lib_runtime_path.c_str()); + } else { + GGMLHEXAGON_LOG_ERROR("setenv LD_LIBRARY_PATH %s failure", lib_runtime_path.c_str()); + } + + std::string adsp_runtime_path = path + ";/usr/local/lib;/usr/lib"; + if (0 == setenv("ADSP_LIBRARY_PATH", adsp_runtime_path.c_str(), 1)) { + GGMLHEXAGON_LOG_DEBUG("setenv ADSP_LIBRARY_PATH %s successfully", adsp_runtime_path.c_str()); + } else { + GGMLHEXAGON_LOG_ERROR("setenv ADSP_LIBRARY_PATH %s failure", adsp_runtime_path.c_str()); + } +#elif defined(_WIN32) + // WoA: PATH uses ';' as separator + std::string lib_runtime_path = path + ";C:\\Windows\\System32;C:\\Windows\\SysWOW64"; + if (0 == _putenv_s("PATH", lib_runtime_path.c_str())) { + GGMLHEXAGON_LOG_DEBUG("setenv PATH %s successfully", lib_runtime_path.c_str()); + } else { + GGMLHEXAGON_LOG_ERROR("setenv PATH %s failure", lib_runtime_path.c_str()); + } + + std::string adsp_runtime_path = path + ";C:\\Windows\\System32"; + if (0 == _putenv_s("ADSP_LIBRARY_PATH", adsp_runtime_path.c_str())) { + GGMLHEXAGON_LOG_DEBUG("setenv ADSP_LIBRARY_PATH %s successfully", adsp_runtime_path.c_str()); + } else { + GGMLHEXAGON_LOG_ERROR("setenv ADSP_LIBRARY_PATH %s failure", adsp_runtime_path.c_str()); + } +#endif +} + +static inline bool ggml_hexagon_is_repack_type(enum ggml_type type) { + return type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 || + type == GGML_TYPE_Q8_0 || type == GGML_TYPE_IQ4_NL || + type == GGML_TYPE_MXFP4 || type == GGML_TYPE_Q5_K; +} + +// Some weight types are stored in the repack buffer in a different format +// than their logical ggml type (BF16 as F16, Q4_K/Q5_K/Q6_K as Q4_0); the DSP +// kernels only see the storage type. Q4_K/Q5_K/Q6_K are stored as Q4_0 (not +// Q8_0) so the bandwidth-bound lm-head matvec moves less data per token. +static inline enum ggml_type ggml_hexagon_weight_dsp_type(enum ggml_type type) { + if (type == GGML_TYPE_BF16) return GGML_TYPE_F16; + if (type == GGML_TYPE_Q4_K) return GGML_TYPE_Q4_0; + if (type == GGML_TYPE_Q5_K) return GGML_TYPE_Q4_0; + if (type == GGML_TYPE_Q6_K) return GGML_TYPE_Q4_0; + return type; +} + +static inline bool ggml_hexagon_is_hmx_weight_type(enum ggml_type type) { + return type == GGML_TYPE_F16 || type == GGML_TYPE_F32 || ggml_hexagon_is_repack_type(type); +} + +// ================================================================================================= +// section-6: CDSP helper functions +// ================================================================================================= +static const char * ggmlhexagon_get_dsp_name(int domain_id) { + (void)domain_id; + return "HTP"; +} + +static int ggmlhexagon_get_vtcm_info(int domain, uint32_t attr, uint32_t * capability) { + int hexagon_error = AEE_SUCCESS; + *capability = 0; + + if (attr != VTCM_PAGE && attr != VTCM_COUNT) { + hexagon_error = AEE_EBADPARM; + GGMLHEXAGON_LOG_DEBUG("unsupported attr, only VTCM_PAGE and VTCM_COUNT supported"); + goto bail; + } + + if (remote_handle_control) { + if (domain == CDSP_DOMAIN_ID) { + struct remote_dsp_capability dsp_capability_vtcm_dsp; + dsp_capability_vtcm_dsp.domain = (uint32_t)domain; + dsp_capability_vtcm_dsp.attribute_ID = attr; + dsp_capability_vtcm_dsp.capability = (uint32_t)0; + hexagon_error = remote_handle_control(DSPRPC_GET_DSP_INFO, &dsp_capability_vtcm_dsp, sizeof(struct remote_dsp_capability)); + if ((hexagon_error & 0xFF) == (AEE_EUNSUPPORTEDAPI & 0xFF)) { + GGMLHEXAGON_LOG_DEBUG("FastRPC Capability API is not supported on this device"); + GGMLHEXAGON_LOG_DEBUG("running the use case without checking the capability"); + hexagon_error = AEE_SUCCESS; + goto bail; + } else if (hexagon_error == AEE_SUCCESS) { + *capability = dsp_capability_vtcm_dsp.capability; + } else { + GGMLHEXAGON_LOG_DEBUG("get_vtcm_info failed with error 0x%x", hexagon_error); + goto bail; + } + } else { + hexagon_error = AEE_EUNSUPPORTED; + GGMLHEXAGON_LOG_DEBUG("unsupported domain %d", domain); + goto bail; + } + } else { + hexagon_error = AEE_EUNSUPPORTEDAPI; + GGMLHEXAGON_LOG_DEBUG("remote_dsp_capability interface is not supported on this device"); + } + +bail: + return hexagon_error; +} + +static bool ggmlhexagon_is_unsignedpd_supported(int domain_id) { + int hexagon_error = AEE_SUCCESS; + if (remote_handle_control) { + struct remote_dsp_capability dsp_capability_domain = {static_cast(domain_id), UNSIGNED_PD_SUPPORT, 0}; + hexagon_error = remote_handle_control(DSPRPC_GET_DSP_INFO, &dsp_capability_domain, sizeof(struct remote_dsp_capability)); + if ((hexagon_error & 0xFF) == (AEE_EUNSUPPORTEDAPI & 0xFF)) { + GGMLHEXAGON_LOG_WARN("FastRPC Capability API is not supported on this device. Falling back to signed pd"); + return false; + } + + if (hexagon_error) { + GGMLHEXAGON_LOG_WARN("error 0x%x: FastRPC Capability API failed. falling back to signed pd", hexagon_error); + return false; + } + + if (dsp_capability_domain.capability == 1) { + return true; + } + } else { + hexagon_error = AEE_EUNSUPPORTEDAPI; + GGMLHEXAGON_LOG_WARN("remote_dsp_capability interface is not supported on this device.falling back to signed pd"); + return false; + } + + return false; +} + +static bool ggmlhexagon_is_async_fastrpc_supported(int domain) { + int hexagon_error = AEE_SUCCESS; + if (remote_handle_control) { + if (domain == CDSP_DOMAIN_ID) { + struct remote_dsp_capability dsp_capability_async_support; + dsp_capability_async_support.domain = (uint32_t)domain; + dsp_capability_async_support.attribute_ID = ASYNC_FASTRPC_SUPPORT; + dsp_capability_async_support.capability = (uint32_t)0; + hexagon_error = remote_handle_control(DSPRPC_GET_DSP_INFO, &dsp_capability_async_support, sizeof(struct remote_dsp_capability)); + if ((hexagon_error & 0xFF) == (AEE_EUNSUPPORTEDAPI & 0xFF)) { + GGMLHEXAGON_LOG_WARN("FastRPC Capability API is not supported on this device"); + hexagon_error = AEE_SUCCESS; + goto bail; + } else if (dsp_capability_async_support.capability == 1) { + return true; + } + + if (hexagon_error != AEE_SUCCESS){ + GGMLHEXAGON_LOG_WARN("failed with error 0x%x", hexagon_error); + goto bail; + } + } else { + hexagon_error = AEE_EUNSUPPORTED; + GGMLHEXAGON_LOG_WARN("async FastRPC is not supported on domain %d", domain); + goto bail; + } + } else { + hexagon_error = AEE_EUNSUPPORTEDAPI; + GGMLHEXAGON_LOG_WARN("remote_dsp_capability interface is not supported on this device"); + } + +bail: + return false; +} + +// Probe whether the DSP domain supports mapping buffers into the extended +// (>=4 GiB) virtual address range. If 1, fastrpc_mmap can use +// FASTRPC_MAP_FD_EXTENDED / FASTRPC_MAP_FD_DELAYED_EXTENDED to place buffers +// beyond the 32-bit user VA ceiling, unlocking scatter-gather and sliding +// window schemes that would otherwise hit the 4 GiB VA hard cap. +static bool ggmlhexagon_is_extended_map_supported(int domain) { + int hexagon_error = AEE_SUCCESS; + if (remote_handle_control) { + if (domain == CDSP_DOMAIN_ID) { + struct remote_dsp_capability dsp_capability_extended_map; + dsp_capability_extended_map.domain = (uint32_t)domain; + dsp_capability_extended_map.attribute_ID = EXTENDED_MAP_SUPPORT; + dsp_capability_extended_map.capability = (uint32_t)0; + hexagon_error = remote_handle_control(DSPRPC_GET_DSP_INFO, &dsp_capability_extended_map, sizeof(struct remote_dsp_capability)); + if ((hexagon_error & 0xFF) == (AEE_EUNSUPPORTEDAPI & 0xFF)) { + GGMLHEXAGON_LOG_WARN("FastRPC Capability API is not supported on this device"); + hexagon_error = AEE_SUCCESS; + goto bail; + } else if (dsp_capability_extended_map.capability == 1) { + return true; + } + + if (hexagon_error != AEE_SUCCESS){ + GGMLHEXAGON_LOG_WARN("failed with error 0x%x", hexagon_error); + goto bail; + } + } else { + hexagon_error = AEE_EUNSUPPORTED; + GGMLHEXAGON_LOG_WARN("extended VA mapping is not supported on domain %d", domain); + goto bail; + } + } else { + hexagon_error = AEE_EUNSUPPORTEDAPI; + GGMLHEXAGON_LOG_WARN("remote_dsp_capability interface is not supported on this device"); + } + +bail: + return false; +} + +static void ggmlhexagon_set_rpc_latency(remote_handle64 handle, int qos, int latency) { + int hexagon_error = AEE_SUCCESS; + (void)latency; + + if (remote_handle_control) { + // Align with QCOM reference: only enable QoS mode, let DSP decide latency. + struct remote_rpc_control_latency data; + memset(&data, 0, sizeof(data)); + data.enable = qos; + hexagon_error = remote_handle64_control(handle, DSPRPC_CONTROL_LATENCY, (void*)&data, sizeof(data)); + if (hexagon_error != AEE_SUCCESS) { + GGMLHEXAGON_LOG_WARN("failed with error 0x%x", hexagon_error); + goto bail; + } else { + GGMLHEXAGON_LOG_ALWAYS("set rpc qos %d (DSP default latency)", qos); + } + } else { + hexagon_error = AEE_EUNSUPPORTEDAPI; + GGMLHEXAGON_LOG_WARN("remote_dsp_capability interface is not supported on this device"); + } + +bail: + return; +} + +static int ggmlhexagon_get_hmx_support_info(int domain, uint32_t attr, uint32_t * capability) { + int hexagon_error = AEE_SUCCESS; + *capability = 0; + + if (attr != HMX_SUPPORT_SPATIAL && attr != HMX_SUPPORT_DEPTH) { + hexagon_error = AEE_EBADPARM; + GGMLHEXAGON_LOG_WARN("unsupported attr, only HMX_SUPPORT_SPATIAL and HMX_SUPPORT_DEPTH supported"); + goto bail; + } + + if (remote_handle_control) { + if (domain == CDSP_DOMAIN_ID) { + struct remote_dsp_capability dsp_capability_hmx_dsp; + dsp_capability_hmx_dsp.domain = (uint32_t)domain; + dsp_capability_hmx_dsp.attribute_ID = attr; + dsp_capability_hmx_dsp.capability = (uint32_t)0; + hexagon_error = remote_handle_control(DSPRPC_GET_DSP_INFO, &dsp_capability_hmx_dsp, sizeof(struct remote_dsp_capability)); + if ((hexagon_error & 0xFF) == (AEE_EUNSUPPORTEDAPI & 0xFF)) { + GGMLHEXAGON_LOG_DEBUG("FastRPC Capability API is not supported on this device"); + hexagon_error = AEE_SUCCESS; + goto bail; + } + else if (hexagon_error == AEE_SUCCESS) { + *capability = dsp_capability_hmx_dsp.capability; + } else { + GGMLHEXAGON_LOG_DEBUG("get_hmx_support_info failed with Error 0x%x", hexagon_error); + goto bail; + } + } else { + hexagon_error = AEE_EUNSUPPORTED; + GGMLHEXAGON_LOG_DEBUG("HMX support is not there for domain %d", domain); + goto bail; + } + } else { + hexagon_error = AEE_EUNSUPPORTEDAPI; + GGMLHEXAGON_LOG_DEBUG("remote_dsp_capability interface is not supported on this device"); + } + +bail: + return hexagon_error; +} + +static int ggmlhexagon_get_hvx_arch_ver(int domain, uint32_t * capability) { + int hexagon_error = AEE_SUCCESS; + *capability = 0; + if(remote_handle_control) { + struct remote_dsp_capability dsp_capability_arch_ver; + dsp_capability_arch_ver.domain = (uint32_t)domain; + dsp_capability_arch_ver.attribute_ID = ARCH_VER; + dsp_capability_arch_ver.capability = (uint32_t)0; + hexagon_error = remote_handle_control(DSPRPC_GET_DSP_INFO, &dsp_capability_arch_ver, sizeof(struct remote_dsp_capability)); + if ((hexagon_error & 0xFF) == (AEE_EUNSUPPORTEDAPI & 0xFF)) { + GGMLHEXAGON_LOG_DEBUG("FastRPC Capability API is not supported on this device"); + hexagon_error = AEE_SUCCESS; + goto bail; + } else if (hexagon_error == AEE_SUCCESS) { + *capability = dsp_capability_arch_ver.capability & 0xFF; + } else { + GGMLHEXAGON_LOG_DEBUG("get_hex_arch_ver failed with error 0x%x", hexagon_error); + goto bail; + } + } else { + hexagon_error = AEE_EUNSUPPORTEDAPI; + GGMLHEXAGON_LOG_DEBUG("remote_dsp_capability interface is not supported on this device"); + } + +bail: + return hexagon_error; +} + +static int ggmlhexagon_get_hvx_support_info(int domain, uint32_t attr, uint32_t * capability) { + int hexagon_error = AEE_SUCCESS; + *capability = 0; + if (attr == HVX_SUPPORT_64B) { + hexagon_error = AEE_EBADPARM; + GGMLHEXAGON_LOG_DEBUG("latest targets have 128 byte HVX register, use HVX_SUPPORT_128B instead of HVX_SUPPORT_64B"); + goto bail; + } + + if (attr != HVX_SUPPORT_128B) { + hexagon_error = AEE_EBADPARM; + GGMLHEXAGON_LOG_DEBUG("unsupported attr. only HVX_SUPPORT_128B supported"); + goto bail; + } + + if (remote_handle_control) { + if (domain == CDSP_DOMAIN_ID) { + struct remote_dsp_capability dsp_capability_hvx_dsp; + dsp_capability_hvx_dsp.domain = (uint32_t)domain; + dsp_capability_hvx_dsp.attribute_ID = attr; + dsp_capability_hvx_dsp.capability = (uint32_t)0; + hexagon_error = remote_handle_control(DSPRPC_GET_DSP_INFO, &dsp_capability_hvx_dsp, sizeof(struct remote_dsp_capability)); + if ((hexagon_error & 0xFF)==(AEE_EUNSUPPORTEDAPI & 0xFF)) { + GGMLHEXAGON_LOG_DEBUG("FastRPC Capability API is not supported on this device"); + hexagon_error = AEE_SUCCESS; + goto bail; + } else if (hexagon_error == AEE_SUCCESS) { + *capability = dsp_capability_hvx_dsp.capability; + } else { + GGMLHEXAGON_LOG_DEBUG("failed with error 0x%x", hexagon_error); + goto bail; + } + } else { + hexagon_error = AEE_EUNSUPPORTED; + GGMLHEXAGON_LOG_DEBUG("HVX support is not available on domain %d", domain); + goto bail; + } + } else { + hexagon_error = AEE_EUNSUPPORTEDAPI; + GGMLHEXAGON_LOG_DEBUG("remote_dsp_capability interface is not supported on this device"); + } + +bail: + return hexagon_error; +} + +static int ggmlhexagon_init_rpcmempool(ggml_backend_hexagon_context * ctx) { + if (nullptr == ctx) { + GGMLHEXAGON_LOG_ERROR("sanity check failure"); + return 1; + } + + int htp_arch = 0; + htp_arch = ggmlhexagon_probe_dspinfo(ctx); + if (0 == htp_arch) { + GGMLHEXAGON_LOG_ERROR("failed to get valid htp arch"); + return 2; + } + + size_t targets_79plus[] = { 4032, 3968, 3840, 3072, 2048 }; + size_t targets_legacy[] = { 3830, 3072, 2048 }; + size_t * targets = (htp_arch > 75) ? targets_79plus : targets_legacy; + size_t n_targets = (htp_arch > 75) ? sizeof(targets_79plus) / sizeof(targets_79plus[0]) + : sizeof(targets_legacy) / sizeof(targets_legacy[0]); + for (size_t i = 0; i < n_targets; i++) { + size_t pool_size = targets[i] - 8; + uint8_t * buf = static_cast(rpcmem_alloc2(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, pool_size * SIZE_IN_MB)); + if (buf) { + ctx->rpc_mempool = buf; + ctx->rpc_mempool_capacity = targets[i] * SIZE_IN_MB; + ctx->rpc_mempool_len = pool_size * SIZE_IN_MB; + GGMLHEXAGON_LOG_ALWAYS("rpc mempool: %zu MiB allocated (target %zu MiB, device %d)", + pool_size, targets[i], ctx->device); + goto mempool_acquired; + } + GGMLHEXAGON_LOG_DEBUG("init rpc mempool: alloc %zu MiB failed, trying next smaller target", pool_size); + } + GGMLHEXAGON_LOG_ERROR("init rpc mempool: all allocation attempts failure"); + return 3; + +mempool_acquired: + ctx->rpc_mempool_handle = rpcmem_to_fd(ctx->rpc_mempool); + if (ctx->rpc_mempool_handle < 0) { + GGMLHEXAGON_LOG_ERROR("rpcmem_to_fd failed for %p", ctx->rpc_mempool); + rpcmem_free(ctx->rpc_mempool); + ctx->rpc_mempool = nullptr; + return 4; + } + GGMLHEXAGON_LOG_INFO("rpc mempool handle %d", ctx->rpc_mempool_handle); + GGMLHEXAGON_LOG_INFO("rpc mempool addr %p", ctx->rpc_mempool); + GGMLHEXAGON_LOG_INFO("rpc mempool size %lld(%dMB)", ctx->rpc_mempool_len, ctx->rpc_mempool_len / SIZE_IN_MB); + // Register mempool with FastRPC kernel driver. + // rpc_mmap_mode = 0: FASTRPC_MAP_FD_DELAYED (default), defers DSP-side mapping until HAP_mmap2(). + // rpc_mmap_mode = 1: FASTRPC_MAP_FD (eager), creates immediate kernel-level mapping and pins pages. + enum fastrpc_map_flags mmap_flags = (g_hexagon_appcfg.rpc_mmap_mode == 1) + ? FASTRPC_MAP_FD + : FASTRPC_MAP_FD_DELAYED; + const char * mmap_mode_str = (g_hexagon_appcfg.rpc_mmap_mode == 1) ? "EAGER" : "DELAYED"; + int mmap_err = fastrpc_mmap(ctx->domain_id, ctx->rpc_mempool_handle, ctx->rpc_mempool, 0, ctx->rpc_mempool_len, mmap_flags); + if (mmap_err != 0) { + GGMLHEXAGON_LOG_ERROR("fastrpc_mmap(%s) returned %d (fd=%d), aborting backend init", + mmap_mode_str, mmap_err, ctx->rpc_mempool_handle); + rpcmem_free(ctx->rpc_mempool); + ctx->rpc_mempool = nullptr; + return 5; + } + GGMLHEXAGON_LOG_INFO("fastrpc_mmap(%s) OK: fd=%d, size=%dMB", + mmap_mode_str, ctx->rpc_mempool_handle, ctx->rpc_mempool_len / SIZE_IN_MB); + + // Register mempool on DSP side via pure-scalar IDL call. + // This avoids FastRPC's fdlist_fd_from_buf() scan that triggers + // implicit fd_mmap_create when dsptensor.data pointers are passed. + // The DSP will call HAP_mmap2(fd) to get a user-space-accessible VA. + uint32_t ion_fd = (uint32_t)ctx->rpc_mempool_handle; + uint32_t size_lo = (uint32_t)(ctx->rpc_mempool_len & 0xFFFFFFFF); + uint32_t size_hi = (uint32_t)((ctx->rpc_mempool_len >> 32) & 0xFFFFFFFF); + + int64_t t0_reg = ggml_time_us(); + int reg_err = ggml_dsp_register_rpcmem(ctx->ggmlop_handle, ion_fd, size_lo, size_hi); + int64_t dt_reg = ggml_time_us() - t0_reg; + if (reg_err != AEE_SUCCESS) { + GGMLHEXAGON_LOG_ERROR("dsp_register_rpcmem failed: 0x%x, aborting backend init", reg_err); + fastrpc_munmap(ctx->domain_id, ctx->rpc_mempool_handle, ctx->rpc_mempool, ctx->rpc_mempool_len); + rpcmem_free(ctx->rpc_mempool); + ctx->rpc_mempool = nullptr; + return 6; + } + GGMLHEXAGON_LOG_ALWAYS("registered mempool base via scalar call: fd=%d, size=%dMB, time=%lld us", + ctx->rpc_mempool_handle, ctx->rpc_mempool_len / SIZE_IN_MB, (long long)dt_reg); + GGMLHEXAGON_LOG_ALWAYS("mempool layout: total=%zuMB", ctx->rpc_mempool_len / SIZE_IN_MB); + // Prime the RPC overhead profiler with a few no-op warmup calls to get + // a meaningful min/max/avg distribution without polluting the pool. + for (int i = 0; i < 6; i++) { + (void)hexagon_warmup_invoke_timed(ctx); + } + + return 0; +} + +static void ggmlhexagon_deinit_rpcmempool(ggml_backend_hexagon_context * ctx) { + if (ctx->rpc_mempool) { + GGMLHEXAGON_LOG_DEBUG("free rpc mempool %p", ctx->rpc_mempool); + fastrpc_munmap(ctx->domain_id, ctx->rpc_mempool_handle, ctx->rpc_mempool, ctx->rpc_mempool_len); + rpcmem_free(ctx->rpc_mempool); + ctx->rpc_mempool = nullptr; + ctx->rpc_mempool_len = 0; + ctx->rpc_mempool_capacity = 0; + } +} + +static int ggmlhexagon_probe_dspinfo(ggml_backend_hexagon_context * ctx) { + if (ctx == nullptr) { + return 0; + } + + int htp_arch = 0; + uint32_t dsp_version = 0; + ggmlhexagon_get_hvx_arch_ver(ctx->domain_id, &dsp_version); + if (dsp_version == 0x68 || dsp_version == 0x69 || dsp_version == 0x73 + || dsp_version == 0x75 || dsp_version == 0x79 || dsp_version == 0x81) { + //0x68 -> 68, 0x69 -> 69, 0x73 -> 73, 0x75 -> 75, 0x79 -> 79, 0x81 -> 81 + htp_arch = ggmlhexagon_htparch_hex_to_decimal(dsp_version); + struct qcom_socinfo * socinfo = ggmlhexagon_get_socinfo_from_htparch(htp_arch); + GGML_ASSERT(nullptr != socinfo); + ctx->socinfo = *socinfo; + size_t total_mem = ggmlhexagon_get_system_total_memory_in_bytes(); + GGMLHEXAGON_LOG_ALWAYS("device info: %s, %s, dsp arch version 0x%x, system mem size %d MiB", + socinfo->soc_desc, ggmlhexagon_get_htparch_desc(htp_arch), dsp_version, total_mem / SIZE_IN_MB); + } else { + GGMLHEXAGON_LOG_ERROR("device info: unknown"); + GGML_ASSERT(1 == 0); + } + + uint32_t vtcm_count = 0; + uint32_t vtcm_page = 0; + ggmlhexagon_get_vtcm_info(ctx->domain_id, VTCM_COUNT, &vtcm_count); + ggmlhexagon_get_vtcm_info(ctx->domain_id, VTCM_PAGE, &vtcm_page); + ctx->has_vtcm = (vtcm_count > 0 && vtcm_page > 0); + + uint32_t hmx_depth = 0; + uint32_t hmx_spatial = 0; + //FIXME: better approach to get correct/accurate info + ggmlhexagon_get_hmx_support_info(ctx->domain_id, HMX_SUPPORT_DEPTH, &hmx_depth); + ggmlhexagon_get_hmx_support_info(ctx->domain_id, HMX_SUPPORT_SPATIAL, &hmx_spatial); + + uint32_t hvx_support_128b = 0; + ggmlhexagon_get_hvx_support_info(ctx->domain_id, HVX_SUPPORT_128B, &hvx_support_128b); + ctx->has_hvx = (hvx_support_128b > 0); + ctx->has_hmx = (hmx_depth > 0 || hmx_spatial > 0); + // Fallback: DSPRPC_GET_DSP_INFO may not report HMX on some devices + // HMX is present on V73+ (Snapdragon 8 Gen 2 and later); the DSP skel is built with -mhmx. + if (!ctx->has_hmx && htp_arch >= V73) { + ctx->has_hmx = true; + } + GGMLHEXAGON_LOG_DEBUG("dsp arch version %d, vtcm_count %d, vtcm_page %d", htp_arch, vtcm_count, vtcm_page); + //FIXME: hmx_depth/hmx_spatial report 0 via DSPRPC_GET_DSP_INFO on some devices + ctx->has_async_fastrpc = ggmlhexagon_is_async_fastrpc_supported(ctx->domain_id); + ctx->has_extended_map = ggmlhexagon_is_extended_map_supported(ctx->domain_id); + GGMLHEXAGON_LOG_ALWAYS("device %d caps: has_vtcm=%d,has_hvx=%d,has_hmx=%d,hvx_support_128b %d," + "unsigned pd supported %d, async fastrpc supported %d, extended va map supported %d", + ctx->device, (int)ctx->has_vtcm, (int)ctx->has_hvx, (int)ctx->has_hmx, hvx_support_128b, + ggmlhexagon_is_unsignedpd_supported(ctx->domain_id), + (int)ctx->has_async_fastrpc, + (int)ctx->has_extended_map); + return htp_arch; +} + +static void ggmlhexagon_deinit_cdsp(ggml_backend_hexagon_context * ctx) { + GGMLHEXAGON_LOG_ALWAYS("enter %s", __FUNCTION__); + int hexagon_error = AEE_SUCCESS; + if (ctx->ggmlop_handle != 0) { + hexagon_error = ggml_dsp_close(ctx->ggmlop_handle); + if (AEE_SUCCESS != hexagon_error) { + GGMLHEXAGON_LOG_ERROR("error 0x%x: failed to close ggmlop dsp handle", hexagon_error); + } + ctx->ggmlop_handle = 0; + } + ggmlhexagon_deinit_rpcmempool(ctx); + //probe before domain_id is invalidated so AP-side domain queries still work + if (ctx->domain_id >= 0) { + ggmlhexagon_probe_dspinfo(ctx); + } + ggmlhexagon_dump_perf_stats(ctx); + ctx->domain_id = -1; + GGMLHEXAGON_LOG_ALWAYS("leave %s", __FUNCTION__); +} + +static int ggmlhexagon_init_dsp(ggml_backend_hexagon_context * ctx) { + GGMLHEXAGON_LOG_ALWAYS("enter %s", __FUNCTION__); + int htp_arch = 0; + int hexagon_error = AEE_SUCCESS; + int domain_id = CDSP_DOMAIN_ID; + bool got_uri = false; + const char * uri = NULL; + domain * my_domain = NULL; + bool is_unsignedpd_enabled = false; + char final_uri[512]; + char htp_uri[256]; + + if (nullptr == ctx) + return 1; + if (0 != ctx->ggmlop_handle) { + GGMLHEXAGON_LOG_DEBUG("already init Hexagon CDSP with backend %d(%s)", ctx->device, ctx->name); + return 0; + } + if (!remote_session_control) { + GGMLHEXAGON_LOG_ERROR("remote_session_control not available"); + hexagon_error = AEE_EUNSUPPORTED; + goto bail; + } + + GGMLHEXAGON_LOG_DEBUG("init Hexagon CDSP with backend %d(%s)", ctx->device, ctx->name); + ctx->ggmlop_handle = 0; + my_domain = htpdrv_get_domain(domain_id); + if (NULL == my_domain) { + GGMLHEXAGON_LOG_ERROR("unable to get domain struct %d", domain_id); + goto bail; + } + uri = my_domain->uri; + GGMLHEXAGON_LOG_DEBUG("domain uri=%s", uri); + // Reserve new FastRPC session (PD) for additional devices (dev_id > 0) + // dev_id == 0 reuses the default CDSP PD (session_id=0) + ctx->session_id = 0; + if (ctx->device > 0) { + struct remote_rpc_reserve_new_session n; + n.domain_name_len = strlen(CDSP_DOMAIN_NAME); + n.domain_name = const_cast(CDSP_DOMAIN_NAME); + char sess_name[32]; + snprintf(sess_name, sizeof(sess_name), "HTP%d", ctx->device); + n.session_name = sess_name; + n.session_name_len = strlen(sess_name); + + int err = remote_session_control(FASTRPC_RESERVE_NEW_SESSION, (void *) &n, sizeof(n)); + if (err != AEE_SUCCESS) { + GGMLHEXAGON_LOG_WARN("FASTRPC_RESERVE_NEW_SESSION failed for device %d: error 0x%x", ctx->device, err); + hexagon_error = err; + goto bail; + } + ctx->session_id = n.session_id; + domain_id = n.effective_domain_id; + GGMLHEXAGON_LOG_VERBOSE("reserved new session: device=%d session_id=%d effective_domain_id=%d", + ctx->device, ctx->session_id, domain_id); + } + + is_unsignedpd_enabled = ggmlhexagon_is_unsignedpd_supported(domain_id); + if (!is_unsignedpd_enabled) { + GGMLHEXAGON_LOG_ERROR("unsigned PD not allowed on domain %d, using signed offload", domain_id); + goto bail; + } + + ctx->domain_id = domain_id; + GGMLHEXAGON_LOG_ALWAYS("using Hexagon domain %d(%s)", domain_id, ggmlhexagon_get_dsp_name(domain_id)); + GGMLHEXAGON_LOG_ALWAYS("unsignedpd_enabled %d", is_unsignedpd_enabled); + if (is_unsignedpd_enabled) { + struct remote_rpc_control_unsigned_module data; + data.enable = 1; + data.domain = domain_id; + hexagon_error = remote_session_control(DSPRPC_CONTROL_UNSIGNED_MODULE, (void *)&data, sizeof(data)); + GGMLHEXAGON_LOG_DEBUG("remote_session_control returned %d for configuring unsigned PD", hexagon_error); + if (AEE_SUCCESS != hexagon_error) { + GGMLHEXAGON_LOG_ERROR("error 0x%x: remote_session_control failed", hexagon_error); + goto bail; + } + } + + // Probe arch and build the versioned dsp skel URI + htp_arch = ggmlhexagon_probe_dspinfo(ctx); + GGML_ASSERT(0 != htp_arch); + + snprintf(htp_uri, sizeof(htp_uri), + "file:///libggml-htp-v%u.so?ggml_dsp_skel_handle_invoke&_modver=1.0&_idlver=" GGML_DSP_IDL_VERSION, + htp_arch); + + // Build the final URI for ggml_dsp_open. + // session_id > 0: use FASTRPC_GET_URI to obtain the session-specific URI. + // session_id == 0 (or FASTRPC_GET_URI failure): concatenate htp_uri + domain uri. + if (ctx->session_id > 0) { + struct remote_rpc_get_uri u = {}; + u.session_id = ctx->session_id; + u.domain_name = const_cast(CDSP_DOMAIN_NAME); + u.domain_name_len = strlen(CDSP_DOMAIN_NAME); + u.module_uri = const_cast(htp_uri); + u.module_uri_len = strlen(htp_uri); + u.uri = final_uri; + u.uri_len = sizeof(final_uri); + int err = remote_session_control(FASTRPC_GET_URI, (void *) &u, sizeof(u)); + if (err == AEE_SUCCESS) { + got_uri = true; + GGMLHEXAGON_LOG_DEBUG("session URI for session_id=%d: %s", ctx->session_id, final_uri); + } else { + GGMLHEXAGON_LOG_WARN("FASTRPC_GET_URI failed for session_id=%d: error 0x%x, fallback to %s%s", + ctx->session_id, err, htp_uri, uri); + } + } + if (!got_uri) { + snprintf(final_uri, sizeof(final_uri), "%s%s", htp_uri, uri); + } + + GGMLHEXAGON_LOG_DEBUG("ggmlop domain uri: %s", final_uri); + hexagon_error = ggml_dsp_open(final_uri, &ctx->ggmlop_handle); + if (AEE_SUCCESS == hexagon_error) { + GGMLHEXAGON_LOG_ALWAYS("succeed to open domain %d(%s)", domain_id, ggmlhexagon_get_dsp_name(domain_id)); + hexagon_error = ggml_dsp_setclocks(ctx->ggmlop_handle, g_hexagon_appcfg.dump_diag_info, g_hexagon_appcfg.thread_counts, &ctx->dsp_thread_counts); + if (AEE_SUCCESS != hexagon_error) { + GGMLHEXAGON_LOG_ERROR("ggml_dsp_setclocks failed: 0x%x", hexagon_error); + goto bail; + } + // Mirror DSP-side clamp into the global cfg so subsequent log sites + // (including the dtor, where ctx may be unavailable) reflect the + // real thread count in effect rather than the user's requested hint. + // Guard: on RPC failure dsp_thread_counts stays 0; keep cfg value. + if (ctx->dsp_thread_counts > 0) { + g_hexagon_appcfg.thread_counts = ctx->dsp_thread_counts; + ctx->n_threads = ctx->dsp_thread_counts; + } + ggmlhexagon_set_rpc_latency(ctx->ggmlop_handle, RPC_PM_QOS, 100); + if (0 != ggmlhexagon_init_rpcmempool(ctx)) { + GGMLHEXAGON_LOG_ERROR("failed to init rpc mempool"); + goto bail; + } + // Push DSP-side cache optimization bitmask via execute_batch(0xFFFC) + // special mode (no IDL change). batch_offset = packed payload, batch_size = mode tag. + // Payload bit layout (low bits are dsp_cache_mode): + // bits 0..3 : dsp_cache_mode (first-touch weight / prior-dst skip / bulk dst flush / selective bulk flush) + // bit 16 : dsp_cache_trace_bit0 (1 = emit [DSP-CACHE-TRACE-BIT0] per bit 0 decision) + // bit 17 : dsp_cache_trace_bit1 (1 = emit [DSP-CACHE-TRACE-BIT1] per bit 1 decision) + // See the dsp_cache_mode and dsp_cache_trace_bit{0,1} comments in hexagon_appcfg_t. + // + // Note: must run AFTER ggmlhexagon_init_rpcmempool(). The DSP-side + // 0xFFFC handler in entry.c asserts mempool_dsp_base != NULL; without the + // mempool registered first it returns AEE_EBADPARM (0x8000040e) and + // the bitmask is silently dropped. + { + const uint32_t mode_bits = (uint32_t)g_hexagon_appcfg.dsp_cache_mode & 0xFu; + const uint32_t trace_bit0 = (g_hexagon_appcfg.dsp_cache_trace_bit0 ? 0x10000u : 0u); + const uint32_t trace_bit1 = (g_hexagon_appcfg.dsp_cache_trace_bit1 ? 0x20000u : 0u); + const uint32_t payload = trace_bit1 | trace_bit0 | mode_bits; + int opts_err = ggml_dsp_execute_batch(ctx->ggmlop_handle, payload, 0xFFFC); + if (AEE_SUCCESS != opts_err) { + GGMLHEXAGON_LOG_WARN("set dsp_cache_mode=0x%x + dsp_cache_trace_bit0=%d + dsp_cache_trace_bit1=%d failed: 0x%x (DSP-side optimizations disabled)", + mode_bits, g_hexagon_appcfg.dsp_cache_trace_bit0, + g_hexagon_appcfg.dsp_cache_trace_bit1, opts_err); + g_hexagon_appcfg.dsp_cache_mode = 0; // fall back to baseline + g_hexagon_appcfg.dsp_cache_trace_bit0 = 0; + g_hexagon_appcfg.dsp_cache_trace_bit1 = 0; + } else { + GGMLHEXAGON_LOG_ALWAYS("[AP-CACHE-MODE] dsp_cache_mode=0x%x + dsp_cache_trace_bit0=%d + dsp_cache_trace_bit1=%d pushed to DSP (payload=0x%x)", + mode_bits, g_hexagon_appcfg.dsp_cache_trace_bit0, + g_hexagon_appcfg.dsp_cache_trace_bit1, payload); + } + + /* Warmup FastRPC/mempool path once before first real inference. + * This triggers delayed mempool mapping and touches the DSP entry path + * so the first real batch does not pay cold-state penalty. */ + int warmup_err = ggml_dsp_execute_batch(ctx->ggmlop_handle, 0, 0xFFFB); + if (AEE_SUCCESS != warmup_err) { + GGMLHEXAGON_LOG_ERROR("warmup execute_batch failed: 0x%x", warmup_err); + } else { + GGMLHEXAGON_LOG_ALWAYS("[AP-WARMUP] FastRPC/mempool warmup done"); + } + } + } else { + GGMLHEXAGON_LOG_ERROR("error 0x%x: failed to open domain %d(%s)", hexagon_error, domain_id, ggmlhexagon_get_dsp_name(domain_id)); + goto bail; + } + + snprintf(ctx->name, sizeof(ctx->name), "HTP%d", ctx->device); + GGMLHEXAGON_LOG_ALWAYS("leave %s", __FUNCTION__); + return 0; + +bail: + ggmlhexagon_deinit_cdsp(ctx); + GGMLHEXAGON_LOG_ALWAYS("leave %s", __FUNCTION__); + return -1; +} + +// ================================================================================================= +// section-7: pack & repack help functions +// ================================================================================================= + +// ---- Tiled repack for HVX-quant MUL_MAT ---- +// HVX-quant kernels (hvx_mm_2d_repacked_*_flat etc.) expect tile-based weight +// layout: each 32x32 tile is tile_size bytes, organized as (ct, kt) major with +// (cp, row) minor inside each tile. Standard GGML row-major layout must be +// converted before passing to DSP. + +static void unpack_q4_0_quants(uint8_t * qs, const block_q4_0 * x) { + for (unsigned int i = 0; i < QK4_0 / 2; ++i) { + const int x0 = (x->qs[i] & 0x0F); + const int x1 = (x->qs[i] >> 4); + qs[i + 0] = x0; + qs[i + QK4_0 / 2] = x1; + } +} + +static void unpack_q4_1_quants(uint8_t * qs, const block_q4_1 * x) { + for (unsigned int i = 0; i < QK4_1 / 2; ++i) { + const int x0 = (x->qs[i] & 0x0F); + const int x1 = (x->qs[i] >> 4); + qs[i + 0] = x0; + qs[i + QK4_1 / 2] = x1; + } +} + +static void unpack_mxfp4_quants(uint8_t * qs, const block_mxfp4 * x) { + for (unsigned int i = 0; i < QK_MXFP4 / 2; ++i) { + const int x0 = (x->qs[i] & 0x0F); + const int x1 = (x->qs[i] >> 4); + qs[i + 0] = x0; + qs[i + QK_MXFP4 / 2] = x1; + } +} + +static void pack_mxfp4_quants(block_mxfp4 * x, const uint8_t * qs) { + for (unsigned int i = 0; i < QK_MXFP4 / 2; ++i) { + x->qs[i] = qs[i] | (qs[i + QK_MXFP4 / 2] << 4); + } +} + +static void pack_q4_0_quants(block_q4_0 * x, const uint8_t * qs) { + for (unsigned int i = 0; i < QK4_0 / 2; ++i) { + x->qs[i] = qs[i] | (qs[i + QK4_0 / 2] << 4); + } +} + +static void pack_q4_1_quants(block_q4_1 * x, const uint8_t * qs) { + for (unsigned int i = 0; i < QK4_1 / 2; ++i) { + x->qs[i] = qs[i] | (qs[i + QK4_1 / 2] << 4); + } +} + +static size_t ggml_hexagon_repacked_size(enum ggml_type type, int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3) { + const uint32_t tile_size = htp_mm_get_weight_tile_size((int)ggml_hexagon_weight_dsp_type(type)); + if (tile_size == 0) return 0; + const uint32_t ne0_p = hex_round_up((uint32_t)ne0, 32); + const uint32_t ne1_p = hex_round_up((uint32_t)ne1, 32); + return (size_t)(ne0_p / 32) * (ne1_p / 32) * tile_size * ne2 * ne3; +} + +static void repack_q4_0_tiled_to_buf(const ggml_tensor * t, const void * data, void * dst_buf) { + const block_q4_0 * src_matrix = (const block_q4_0 *) data; + const int64_t ne0 = t->ne[0], ne1 = t->ne[1], ne2 = t->ne[2], ne3 = t->ne[3]; + const int n_col_tiles = hex_round_up((uint32_t)ne1, 32) / 32; + const int n_k_tiles = hex_round_up((uint32_t)ne0, 32) / 32; + const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_0; + const size_t matrix_size = (size_t)n_col_tiles * n_k_tiles * tile_size; + + for (int i3 = 0; i3 < ne3; i3++) { + for (int i2 = 0; i2 < ne2; i2++) { + const block_q4_0 * src_expert = src_matrix + (i3 * ne2 + i2) * (ne1 * (ne0 / 32)); + uint8_t * matrix_dst = (uint8_t *) dst_buf + (i3 * ne2 + i2) * matrix_size; + + for (int ct = 0; ct < n_col_tiles; ct++) { + for (int kt = 0; kt < n_k_tiles; kt++) { + uint8_t * tile_dst = matrix_dst + (ct * n_k_tiles + kt) * tile_size; + + uint8_t tile_quants[32][32]; + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + if (r < ne1 && kt < ne0 / 32) { + unpack_q4_0_quants(tile_quants[row], &src_expert[r * (ne0 / 32) + kt]); + } else { + memset(tile_quants[row], 8, 32); + } + } + + for (int cp = 0; cp < 16; cp++) { + for (int row = 0; row < 32; row++) { + tile_dst[cp * 32 + row] = (tile_quants[row][2 * cp + 1] << 4) | tile_quants[row][2 * cp]; + } + } + + ggml_half * scale_dst = (ggml_half *)(tile_dst + 512); + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + scale_dst[row] = (r < ne1 && kt < ne0 / 32) ? src_expert[r * (ne0 / 32) + kt].d : 0; + } + } + } + } + } +} + +static void repack_q4_1_tiled_to_buf(const ggml_tensor * t, const void * data, void * dst_buf) { + const block_q4_1 * src_matrix = (const block_q4_1 *) data; + const int64_t ne0 = t->ne[0], ne1 = t->ne[1], ne2 = t->ne[2], ne3 = t->ne[3]; + const int n_col_tiles = hex_round_up((uint32_t)ne1, 32) / 32; + const int n_k_tiles = hex_round_up((uint32_t)ne0, 32) / 32; + const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_1; + const size_t matrix_size = (size_t)n_col_tiles * n_k_tiles * tile_size; + + for (int i3 = 0; i3 < ne3; i3++) { + for (int i2 = 0; i2 < ne2; i2++) { + const block_q4_1 * src_expert = src_matrix + (i3 * ne2 + i2) * (ne1 * (ne0 / 32)); + uint8_t * matrix_dst = (uint8_t *) dst_buf + (i3 * ne2 + i2) * matrix_size; + + for (int ct = 0; ct < n_col_tiles; ct++) { + for (int kt = 0; kt < n_k_tiles; kt++) { + uint8_t * tile_dst = matrix_dst + (ct * n_k_tiles + kt) * tile_size; + + uint8_t tile_quants[32][32]; + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + if (r < ne1 && kt < ne0 / 32) { + unpack_q4_1_quants(tile_quants[row], &src_expert[r * (ne0 / 32) + kt]); + } else { + memset(tile_quants[row], 0, 32); + } + } + + for (int cp = 0; cp < 16; cp++) { + for (int row = 0; row < 32; row++) { + tile_dst[cp * 32 + row] = (tile_quants[row][2 * cp + 1] << 4) | tile_quants[row][2 * cp]; + } + } + + ggml_half * scale_dst = (ggml_half *)(tile_dst + 512); + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + if (r < ne1 && kt < ne0 / 32) { + scale_dst[2 * row + 0] = src_expert[r * (ne0 / 32) + kt].d; + scale_dst[2 * row + 1] = src_expert[r * (ne0 / 32) + kt].m; + } else { + scale_dst[2 * row + 0] = 0; + scale_dst[2 * row + 1] = 0; + } + } + } + } + } + } +} + +static void repack_q8_0_tiled_to_buf(const ggml_tensor * t, const void * data, void * dst_buf) { + const block_q8_0 * src_matrix = (const block_q8_0 *) data; + const int64_t ne0 = t->ne[0], ne1 = t->ne[1], ne2 = t->ne[2], ne3 = t->ne[3]; + const int n_col_tiles = hex_round_up((uint32_t)ne1, 32) / 32; + const int n_k_tiles = hex_round_up((uint32_t)ne0, 32) / 32; + const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q8_0; + const size_t matrix_size = (size_t)n_col_tiles * n_k_tiles * tile_size; + + for (int i3 = 0; i3 < ne3; i3++) { + for (int i2 = 0; i2 < ne2; i2++) { + const block_q8_0 * src_expert = src_matrix + (i3 * ne2 + i2) * (ne1 * (ne0 / 32)); + uint8_t * matrix_dst = (uint8_t *) dst_buf + (i3 * ne2 + i2) * matrix_size; + + for (int ct = 0; ct < n_col_tiles; ct++) { + for (int kt = 0; kt < n_k_tiles; kt++) { + uint8_t * tile_dst = matrix_dst + (ct * n_k_tiles + kt) * tile_size; + + for (int cp = 0; cp < 16; cp++) { + int col0 = cp * 2; + int col1 = col0 + 1; + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + const block_q8_0 * b = (r < ne1 && kt < ne0 / 32) ? &src_expert[r * (ne0 / 32) + kt] : NULL; + tile_dst[cp * 64 + 2 * row + 0] = b ? b->qs[col0] : 0; + tile_dst[cp * 64 + 2 * row + 1] = b ? b->qs[col1] : 0; + } + } + + ggml_half * scale_dst = (ggml_half *)(tile_dst + 1024); + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + scale_dst[row] = (r < ne1 && kt < ne0 / 32) ? src_expert[r * (ne0 / 32) + kt].d : 0; + } + } + } + } + } +} + +// Q4_K -> Q4_0 tiled for DSP matmul (lossy conversion, runs once at load). +static void repack_q4k_as_q4_0_tiled_to_buf(const ggml_tensor * t, const void * data, void * dst_buf) { + const int64_t ne0 = t->ne[0], ne1 = t->ne[1], ne2 = t->ne[2], ne3 = t->ne[3]; + const int n_col_tiles = hex_round_up((uint32_t)ne1, 32) / 32; + const int n_k_tiles = hex_round_up((uint32_t)ne0, 32) / 32; + const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_0; + const size_t matrix_size = (size_t)n_col_tiles * n_k_tiles * tile_size; + const int64_t nb_q4 = ne0 / QK4_0; // q4_0 blocks per row + const int64_t nb_q4k = ne0 / QK_K; // q4_K blocks per row + + std::vector row_f32(ne0); + std::vector strip_q4(32 * nb_q4); // canonical q4_0 for one 32-row strip + + for (int i3 = 0; i3 < ne3; i3++) { + for (int i2 = 0; i2 < ne2; i2++) { + const block_q4_K * src_expert = (const block_q4_K *) data + (i3 * ne2 + i2) * (ne1 * nb_q4k); + uint8_t * matrix_dst = (uint8_t *) dst_buf + (i3 * ne2 + i2) * matrix_size; + + for (int ct = 0; ct < n_col_tiles; ct++) { + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r < ne1) { + dequantize_row_q4_K(src_expert + r * nb_q4k, row_f32.data(), ne0); + quantize_row_q4_0_ref(row_f32.data(), strip_q4.data() + row * nb_q4, ne0); + } else { + memset(strip_q4.data() + row * nb_q4, 0, nb_q4 * sizeof(block_q4_0)); + } + } + + for (int kt = 0; kt < n_k_tiles; kt++) { + uint8_t * tile_dst = matrix_dst + (ct * n_k_tiles + kt) * tile_size; + + uint8_t tile_quants[32][32]; + for (int row = 0; row < 32; row++) { + if (kt < nb_q4) { + unpack_q4_0_quants(tile_quants[row], &strip_q4[row * nb_q4 + kt]); + } else { + memset(tile_quants[row], 8, 32); + } + } + + for (int cp = 0; cp < 16; cp++) { + for (int row = 0; row < 32; row++) { + tile_dst[cp * 32 + row] = (tile_quants[row][2 * cp + 1] << 4) | tile_quants[row][2 * cp]; + } + } + + ggml_half * scale_dst = (ggml_half *)(tile_dst + 512); + for (int row = 0; row < 32; row++) { + scale_dst[row] = (kt < nb_q4) ? strip_q4[row * nb_q4 + kt].d : 0; + } + } + } + } + } +} + +// Q6_K weights are converted to Q4_0 (dequant Q6_K -> f32 -> requant Q4_0) +// and stored in the Q4_0 tiled layout, so the DSP reuses the Q4_0 matmul +// kernels. Same approach as Q4_K conversion above. +static void repack_q6k_as_q4_0_tiled_to_buf(const ggml_tensor * t, const void * data, void * dst_buf) { + const int64_t ne0 = t->ne[0], ne1 = t->ne[1], ne2 = t->ne[2], ne3 = t->ne[3]; + const int n_col_tiles = hex_round_up((uint32_t)ne1, 32) / 32; + const int n_k_tiles = hex_round_up((uint32_t)ne0, 32) / 32; + const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_0; + const size_t matrix_size = (size_t)n_col_tiles * n_k_tiles * tile_size; + const int64_t nb_q4 = ne0 / QK4_0; // q4_0 blocks per row + const int64_t nb_q6k = ne0 / QK_K; // q6_K blocks per row + + std::vector row_f32(ne0); + std::vector strip_q4(32 * nb_q4); + + for (int i3 = 0; i3 < ne3; i3++) { + for (int i2 = 0; i2 < ne2; i2++) { + const block_q6_K * src_expert = (const block_q6_K *) data + (i3 * ne2 + i2) * (ne1 * nb_q6k); + uint8_t * matrix_dst = (uint8_t *) dst_buf + (i3 * ne2 + i2) * matrix_size; + + for (int ct = 0; ct < n_col_tiles; ct++) { + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r < ne1) { + dequantize_row_q6_K(src_expert + r * nb_q6k, row_f32.data(), ne0); + quantize_row_q4_0_ref(row_f32.data(), strip_q4.data() + row * nb_q4, ne0); + } else { + memset(strip_q4.data() + row * nb_q4, 0, nb_q4 * sizeof(block_q4_0)); + } + } + + for (int kt = 0; kt < n_k_tiles; kt++) { + uint8_t * tile_dst = matrix_dst + (ct * n_k_tiles + kt) * tile_size; + + uint8_t tile_quants[32][32]; + for (int row = 0; row < 32; row++) { + if (kt < nb_q4) { + unpack_q4_0_quants(tile_quants[row], &strip_q4[row * nb_q4 + kt]); + } else { + memset(tile_quants[row], 8, 32); + } + } + + for (int cp = 0; cp < 16; cp++) { + for (int row = 0; row < 32; row++) { + tile_dst[cp * 32 + row] = (tile_quants[row][2 * cp + 1] << 4) | tile_quants[row][2 * cp]; + } + } + + ggml_half * scale_dst = (ggml_half *)(tile_dst + 512); + for (int row = 0; row < 32; row++) { + scale_dst[row] = (kt < nb_q4) ? strip_q4[row * nb_q4 + kt].d : 0; + } + } + } + } + } +} + +// Q5_K weights are converted to Q4_0 (dequant Q5_K -> f32 -> requant Q4_0) +// and stored in the Q4_0 tiled layout, so the DSP reuses the Q4_0 matmul +// kernels. Same approach as Q4_K/Q6_K conversion above. +static void repack_q5k_as_q4_0_tiled_to_buf(const ggml_tensor * t, const void * data, void * dst_buf) { + const int64_t ne0 = t->ne[0], ne1 = t->ne[1], ne2 = t->ne[2], ne3 = t->ne[3]; + const int n_col_tiles = hex_round_up((uint32_t)ne1, 32) / 32; + const int n_k_tiles = hex_round_up((uint32_t)ne0, 32) / 32; + const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_0; + const size_t matrix_size = (size_t)n_col_tiles * n_k_tiles * tile_size; + const int64_t nb_q4 = ne0 / QK4_0; // q4_0 blocks per row + const int64_t nb_q5k = ne0 / QK_K; // q5_K blocks per row + + std::vector row_f32(ne0); + std::vector strip_q4(32 * nb_q4); + + for (int i3 = 0; i3 < ne3; i3++) { + for (int i2 = 0; i2 < ne2; i2++) { + const block_q5_K * src_expert = (const block_q5_K *) data + (i3 * ne2 + i2) * (ne1 * nb_q5k); + uint8_t * matrix_dst = (uint8_t *) dst_buf + (i3 * ne2 + i2) * matrix_size; + + for (int ct = 0; ct < n_col_tiles; ct++) { + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r < ne1) { + dequantize_row_q5_K(src_expert + r * nb_q5k, row_f32.data(), ne0); + quantize_row_q4_0_ref(row_f32.data(), strip_q4.data() + row * nb_q4, ne0); + } else { + memset(strip_q4.data() + row * nb_q4, 0, nb_q4 * sizeof(block_q4_0)); + } + } + + for (int kt = 0; kt < n_k_tiles; kt++) { + uint8_t * tile_dst = matrix_dst + (ct * n_k_tiles + kt) * tile_size; + + uint8_t tile_quants[32][32]; + for (int row = 0; row < 32; row++) { + if (kt < nb_q4) { + unpack_q4_0_quants(tile_quants[row], &strip_q4[row * nb_q4 + kt]); + } else { + memset(tile_quants[row], 8, 32); + } + } + + for (int cp = 0; cp < 16; cp++) { + for (int row = 0; row < 32; row++) { + tile_dst[cp * 32 + row] = (tile_quants[row][2 * cp + 1] << 4) | tile_quants[row][2 * cp]; + } + } + + ggml_half * scale_dst = (ggml_half *)(tile_dst + 512); + for (int row = 0; row < 32; row++) { + scale_dst[row] = (kt < nb_q4) ? strip_q4[row * nb_q4 + kt].d : 0; + } + } + } + } + } +} + +static void repack_mxfp4_tiled_to_buf(const ggml_tensor * t, const void * data, void * dst_buf) { + const block_mxfp4 * src_matrix = (const block_mxfp4 *) data; + const int64_t ne0 = t->ne[0], ne1 = t->ne[1], ne2 = t->ne[2], ne3 = t->ne[3]; + const int n_col_tiles = hex_round_up((uint32_t)ne1, 32) / 32; + const int n_k_tiles = hex_round_up((uint32_t)ne0, 32) / 32; + const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_MXFP4; + const size_t matrix_size = (size_t)n_col_tiles * n_k_tiles * tile_size; + + for (int i3 = 0; i3 < ne3; i3++) { + for (int i2 = 0; i2 < ne2; i2++) { + const block_mxfp4 * src_expert = src_matrix + (i3 * ne2 + i2) * (ne1 * (ne0 / 32)); + uint8_t * matrix_dst = (uint8_t *) dst_buf + (i3 * ne2 + i2) * matrix_size; + + for (int ct = 0; ct < n_col_tiles; ct++) { + for (int kt = 0; kt < n_k_tiles; kt++) { + uint8_t * tile_dst = matrix_dst + (ct * n_k_tiles + kt) * tile_size; + + uint8_t tile_quants[32][32]; + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + if (r < ne1 && kt < ne0 / 32) { + unpack_mxfp4_quants(tile_quants[row], &src_expert[r * (ne0 / 32) + kt]); + } else { + memset(tile_quants[row], 0, 32); + } + } + + for (int cp = 0; cp < 16; cp++) { + for (int row = 0; row < 32; row++) { + tile_dst[cp * 32 + row] = (tile_quants[row][2 * cp + 1] << 4) | tile_quants[row][2 * cp]; + } + } + + uint8_t * scale_dst = tile_dst + 512; + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + scale_dst[row] = (r < ne1 && kt < ne0 / 32) ? src_expert[r * (ne0 / 32) + kt].e : 0; + } + } + } + } + } +} + +// Inverse repack: convert tiled layout back to canonical GGML layout. +// Used by get_tensor so CPU backends can read weight data in original format. +static void repack_tiled_q4_0_to_buf(void * dst_data, const ggml_tensor * t, size_t size) { + block_q4_0 * dst_matrix = (block_q4_0 *) dst_data; + int64_t ne0 = t->ne[0], ne1 = t->ne[1], ne2 = t->ne[2], ne3 = t->ne[3]; + int64_t ne0_padded = hex_round_up((uint32_t)ne0, 32); + int64_t ne1_padded = hex_round_up((uint32_t)ne1, 32); + int n_col_tiles = ne1_padded / 32; + int n_k_tiles = ne0_padded / 32; + const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_0; + const size_t matrix_size = (size_t)n_col_tiles * n_k_tiles * tile_size; + + for (int i3 = 0; i3 < ne3; i3++) { + for (int i2 = 0; i2 < ne2; i2++) { + block_q4_0 * dst_expert = dst_matrix + (i3 * ne2 + i2) * (ne1 * (ne0 / 32)); + const uint8_t * matrix_src = (const uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size; + + for (int ct = 0; ct < n_col_tiles; ct++) { + for (int kt = 0; kt < n_k_tiles; kt++) { + const uint8_t * tile_src = matrix_src + (ct * n_k_tiles + kt) * tile_size; + + uint8_t tile_quants[32][32]; + for (int cp = 0; cp < 16; cp++) { + for (int row = 0; row < 32; row++) { + uint8_t val = tile_src[cp * 32 + row]; + tile_quants[row][2 * cp + 0] = val & 0x0F; + tile_quants[row][2 * cp + 1] = val >> 4; + } + } + + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + if (r < ne1 && kt < ne0 / 32) { + pack_q4_0_quants(&dst_expert[r * (ne0 / 32) + kt], tile_quants[row]); + } + } + + const ggml_half * scale_src = (const ggml_half *)(tile_src + 512); + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + if (r < ne1 && kt < ne0 / 32) { + dst_expert[r * (ne0 / 32) + kt].d = scale_src[row]; + } + } + } + } + } + } + GGML_UNUSED(size); +} + +static void repack_tiled_q4_1_to_buf(void * dst_data, const ggml_tensor * t, size_t size) { + block_q4_1 * dst_matrix = (block_q4_1 *) dst_data; + int64_t ne0 = t->ne[0], ne1 = t->ne[1], ne2 = t->ne[2], ne3 = t->ne[3]; + int64_t ne0_padded = hex_round_up((uint32_t)ne0, 32); + int64_t ne1_padded = hex_round_up((uint32_t)ne1, 32); + int n_col_tiles = ne1_padded / 32; + int n_k_tiles = ne0_padded / 32; + const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_1; + const size_t matrix_size = (size_t)n_col_tiles * n_k_tiles * tile_size; + + for (int i3 = 0; i3 < ne3; i3++) { + for (int i2 = 0; i2 < ne2; i2++) { + block_q4_1 * dst_expert = dst_matrix + (i3 * ne2 + i2) * (ne1 * (ne0 / 32)); + const uint8_t * matrix_src = (const uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size; + + for (int ct = 0; ct < n_col_tiles; ct++) { + for (int kt = 0; kt < n_k_tiles; kt++) { + const uint8_t * tile_src = matrix_src + (ct * n_k_tiles + kt) * tile_size; + + uint8_t tile_quants[32][32]; + for (int cp = 0; cp < 16; cp++) { + for (int row = 0; row < 32; row++) { + uint8_t val = tile_src[cp * 32 + row]; + tile_quants[row][2 * cp + 0] = val & 0x0F; + tile_quants[row][2 * cp + 1] = val >> 4; + } + } + + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + if (r < ne1 && kt < ne0 / 32) { + pack_q4_1_quants(&dst_expert[r * (ne0 / 32) + kt], tile_quants[row]); + } + } + + const ggml_half * scale_src = (const ggml_half *)(tile_src + 512); + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + if (r < ne1 && kt < ne0 / 32) { + dst_expert[r * (ne0 / 32) + kt].d = scale_src[2 * row + 0]; + dst_expert[r * (ne0 / 32) + kt].m = scale_src[2 * row + 1]; + } + } + } + } + } + } + GGML_UNUSED(size); +} + +// Inverse of repack_q8_0_tiled_to_buf: convert Q8_0 tiled layout back to +// canonical Q8_0 blocks. Used by get_tensor for CPU reference comparison. +static void repack_tiled_q8_0_to_buf(void * dst_data, const ggml_tensor * t, size_t size) { + block_q8_0 * dst_matrix = (block_q8_0 *) dst_data; + int64_t ne0 = t->ne[0], ne1 = t->ne[1], ne2 = t->ne[2], ne3 = t->ne[3]; + int64_t ne0_padded = hex_round_up((uint32_t)ne0, 32); + int64_t ne1_padded = hex_round_up((uint32_t)ne1, 32); + int n_col_tiles = ne1_padded / 32; + int n_k_tiles = ne0_padded / 32; + const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q8_0; + const size_t matrix_size = (size_t)n_col_tiles * n_k_tiles * tile_size; + + for (int i3 = 0; i3 < ne3; i3++) { + for (int i2 = 0; i2 < ne2; i2++) { + block_q8_0 * dst_expert = dst_matrix + (i3 * ne2 + i2) * (ne1 * (ne0 / 32)); + const uint8_t * matrix_src = (const uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size; + + for (int ct = 0; ct < n_col_tiles; ct++) { + for (int kt = 0; kt < n_k_tiles; kt++) { + const uint8_t * tile_src = matrix_src + (ct * n_k_tiles + kt) * tile_size; + + for (int cp = 0; cp < 16; cp++) { + int col0 = cp * 2; + int col1 = col0 + 1; + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + if (r < ne1 && kt < ne0 / 32) { + block_q8_0 & b = dst_expert[r * (ne0 / 32) + kt]; + b.qs[col0] = tile_src[cp * 64 + 2 * row + 0]; + b.qs[col1] = tile_src[cp * 64 + 2 * row + 1]; + } + } + } + + const ggml_half * scale_src = (const ggml_half *)(tile_src + 1024); + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + if (r < ne1 && kt < ne0 / 32) { + dst_expert[r * (ne0 / 32) + kt].d = scale_src[row]; + } + } + } + } + } + } + GGML_UNUSED(size); +} + +// Inverse of repack_mxfp4_tiled_to_buf: convert MXFP4 tiled layout back to +// canonical MXFP4 blocks. Used by get_tensor for CPU reference comparison. +static void repack_tiled_mxfp4_to_buf(void * dst_data, const ggml_tensor * t, size_t size) { + block_mxfp4 * dst_matrix = (block_mxfp4 *) dst_data; + int64_t ne0 = t->ne[0], ne1 = t->ne[1], ne2 = t->ne[2], ne3 = t->ne[3]; + int64_t ne0_padded = hex_round_up((uint32_t)ne0, 32); + int64_t ne1_padded = hex_round_up((uint32_t)ne1, 32); + int n_col_tiles = ne1_padded / 32; + int n_k_tiles = ne0_padded / 32; + const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_MXFP4; + const size_t matrix_size = (size_t)n_col_tiles * n_k_tiles * tile_size; + + for (int i3 = 0; i3 < ne3; i3++) { + for (int i2 = 0; i2 < ne2; i2++) { + block_mxfp4 * dst_expert = dst_matrix + (i3 * ne2 + i2) * (ne1 * (ne0 / 32)); + const uint8_t * matrix_src = (const uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size; + + for (int ct = 0; ct < n_col_tiles; ct++) { + for (int kt = 0; kt < n_k_tiles; kt++) { + const uint8_t * tile_src = matrix_src + (ct * n_k_tiles + kt) * tile_size; + + uint8_t tile_quants[32][32]; + for (int cp = 0; cp < 16; cp++) { + for (int row = 0; row < 32; row++) { + uint8_t val = tile_src[cp * 32 + row]; + tile_quants[row][2 * cp + 0] = val & 0x0F; + tile_quants[row][2 * cp + 1] = val >> 4; + } + } + + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + if (r < ne1 && kt < ne0 / 32) { + pack_mxfp4_quants(&dst_expert[r * (ne0 / 32) + kt], tile_quants[row]); + } + } + + const uint8_t * scale_src = tile_src + 512; + for (int row = 0; row < 32; row++) { + int64_t r = ct * 32 + row; + if (r < ne1 && kt < ne0 / 32) { + dst_expert[r * (ne0 / 32) + kt].e = scale_src[row]; + } + } + } + } + } + } + GGML_UNUSED(size); +} + +// BF16 weights are stored as F16 bytes in the repack buffer so the DSP can +// reuse the F16 matmul kernels. Conversion is exact for values inside the +// F16 exponent range, which always holds for trained projection weights. +static void repack_bf16_to_f16(const ggml_tensor * t, const void * data) { + const int64_t n = ggml_nelements(t); + const ggml_bf16_t * src = (const ggml_bf16_t *) data; + ggml_fp16_t * dst = (ggml_fp16_t *) t->data; + for (int64_t i = 0; i < n; i++) { + dst[i] = ggml_fp32_to_fp16(ggml_bf16_to_fp32(src[i])); + } +} + +// Inverse of repack_bf16_to_f16, used by get_tensor for host read-back. +static void repack_f16_to_bf16(const ggml_tensor * t, void * data, size_t size) { + const int64_t n = (int64_t)(size / sizeof(ggml_bf16_t)); + const ggml_fp16_t * src = (const ggml_fp16_t *) t->data; + ggml_bf16_t * dst = (ggml_bf16_t *) data; + for (int64_t i = 0; i < n; i++) { + dst[i] = ggml_fp32_to_bf16(ggml_fp16_to_fp32(src[i])); + } +} + +// Inverse of repack_q4k_as_q4_0_tiled_to_buf, used by get_tensor for host +// read-back: gather each canonical Q4_0 row from the tiled layout, dequant +// to f32 and requant to Q4_K. Row-at-a-time to keep scratch small. +static void repack_tiled_q4_0_to_q4k_buf(void * dst_data, const ggml_tensor * t, size_t size) { + const int64_t ne0 = t->ne[0], ne1 = t->ne[1], ne2 = t->ne[2], ne3 = t->ne[3]; + const int n_col_tiles = hex_round_up((uint32_t)ne1, 32) / 32; + const int n_k_tiles = hex_round_up((uint32_t)ne0, 32) / 32; + const size_t tile_size = HTP_MM_WEIGHT_TILE_SIZE_Q4_0; + const size_t matrix_size = (size_t)n_col_tiles * n_k_tiles * tile_size; + const int64_t nb_q4 = ne0 / QK4_0; + const int64_t nb_q4k = ne0 / QK_K; + + std::vector row_f32(ne0); + std::vector row_q4(nb_q4); + + for (int i3 = 0; i3 < ne3; i3++) { + for (int i2 = 0; i2 < ne2; i2++) { + const uint8_t * matrix_src = (const uint8_t *) t->data + (i3 * ne2 + i2) * matrix_size; + block_q4_K * dst_expert = (block_q4_K *) dst_data + (i3 * ne2 + i2) * (ne1 * nb_q4k); + + for (int64_t r = 0; r < ne1; r++) { + const int ct = (int)(r / 32); + const int row = (int)(r % 32); + for (int64_t kt = 0; kt < nb_q4; kt++) { + const uint8_t * tile_src = matrix_src + (ct * n_k_tiles + kt) * tile_size; + uint8_t q[32]; + for (int cp = 0; cp < 16; cp++) { + const uint8_t packed = tile_src[cp * 32 + row]; + q[2 * cp + 0] = packed & 0x0F; + q[2 * cp + 1] = packed >> 4; + } + for (int i = 0; i < 16; i++) { + row_q4[kt].qs[i] = (uint8_t)(q[i + 16] << 4) | q[i]; + } + row_q4[kt].d = ((const ggml_half *)(tile_src + 512))[row]; + } + dequantize_row_q4_0(row_q4.data(), row_f32.data(), ne0); + quantize_row_q4_K_ref(row_f32.data(), dst_expert + r * nb_q4k, ne0); + } + } + } + GGML_UNUSED(size); +} + +// ================================================================================================= +// section-8: Qualcomm compatibility layer(ported from Qualcomm's ggml-hexagon) +// ================================================================================================= + +static inline size_t htp_mm_hvx_get_vtcm_sizes( + int kernel_type, int wtype, uint32_t ne10, uint32_t src1_nrows, + uint32_t n_threads, + size_t dst_row_size, size_t src0_row_size, size_t src1_row_size, + uint32_t n_prefetch, + size_t * vtcm_src0_size, size_t * vtcm_src1_size, size_t * vtcm_dst_size +) { + struct htp_mm_hvx_vtcm_layout vtcm_layout; + htp_mm_hvx_vtcm_layout_build(&vtcm_layout, kernel_type, wtype, ne10, src1_nrows, n_threads, + dst_row_size, src0_row_size, src1_row_size, 0, n_prefetch, + false, false, false); + *vtcm_src0_size = vtcm_layout.src0_bytes; + *vtcm_src1_size = vtcm_layout.src1_bytes; + *vtcm_dst_size = vtcm_layout.dst_bytes; + return vtcm_layout.total_bytes; +} + +static inline size_t htp_mm_hvx_id_get_vtcm_sizes( + int wtype, uint32_t ne10, uint32_t src1_nrows, uint32_t n_threads, + size_t src0_row_size, uint32_t n_prefetch, + size_t * vtcm_src0_size, size_t * vtcm_src1_size, size_t * vtcm_dst_size +) { + struct htp_mm_hvx_vtcm_layout vtcm_layout; + htp_mm_hvx_vtcm_layout_build(&vtcm_layout, 0, wtype, ne10, src1_nrows, n_threads, + 0, src0_row_size, 0, 0, n_prefetch, + true, false, false); + *vtcm_src0_size = vtcm_layout.src0_bytes; + *vtcm_src1_size = vtcm_layout.src1_bytes; + *vtcm_dst_size = vtcm_layout.dst_bytes; + return vtcm_layout.total_bytes; +} + +// FA kernel selection: 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported) +// Controlled by ggml-hexagon.cfg: [cdsp] fa_select +static int ggml_hexagon_get_fa_select(void) { + return g_hexagon_appcfg.fa_select; +} + +// Precompute htp_fa_kernel_params on AP side for FLASH_ATTN_EXT. +// Writes to kparams; caller casts from op.kernel_params or a stack local. +// Returns true if a valid kernel (HMX or HVX) was selected. +static bool ggml_hexagon_compute_fa_params( + const ggml_backend_hexagon_context * ctx, + const ggml_tensor * node, + struct htp_fa_kernel_params * kparams +) { + if (ggml_hexagon_get_fa_select() < 1) { + return false; + } + + memset(kparams, 0, sizeof(*kparams)); + + const ggml_tensor * q = node->src[0]; + const ggml_tensor * k = node->src[1]; + const ggml_tensor * v = node->src[2]; + const ggml_tensor * mask = node->src[3]; + const ggml_tensor * dst = node; + + const uint32_t DK = (uint32_t) q->ne[0]; + const uint32_t DV = (uint32_t) v->ne[0]; + const uint32_t neq1 = (uint32_t) q->ne[1]; + const uint32_t nek1 = (uint32_t) k->ne[1]; + const uint32_t n_kv_heads = (uint32_t) k->ne[2]; + const uint32_t G = (uint32_t) q->ne[2] / n_kv_heads; + + float scale = 1.0f, max_bias = 0.0f, logit_softcap = 0.0f; + memcpy(&scale, &node->op_params[0], sizeof(float)); + memcpy(&max_bias, &node->op_params[1], sizeof(float)); + memcpy(&logit_softcap, &node->op_params[2], sizeof(float)); + if (logit_softcap != 0.0f) { + scale /= logit_softcap; + } + + kparams->scale = scale; + kparams->max_bias = max_bias; + kparams->logit_softcap = logit_softcap; + kparams->is_q_fp32 = (q->type == GGML_TYPE_F32) ? 1 : 0; + kparams->is_dst_fp32 = (dst->type == GGML_TYPE_F32) ? 1 : 0; + kparams->G = G; + + const uint32_t n_head = (uint32_t) q->ne[2]; + // largest power of 2 <= n_head + uint32_t n_head_log2 = 1; + while (n_head_log2 * 2u <= n_head) n_head_log2 *= 2; + kparams->n_head_log2 = n_head_log2; + // 2^x = exp(x * ln2), avoiding powf dependency + const float ln2 = 0.6931471805599453f; + kparams->m0 = expf(-ln2 * max_bias / (float) n_head_log2); + kparams->m1 = expf(-ln2 * (max_bias * 0.5f) / (float) n_head_log2); + + // HMX eligibility + bool hmx_eligible = false; + if (ctx->has_hmx && ggml_hexagon_get_fa_select() >= 2 && + k->type == GGML_TYPE_F16 && v->type == GGML_TYPE_F16) { + if (DK % 64 == 0 && DV % 64 == 0 && !(DK <= 128 && neq1 < 5)) { + hmx_eligible = true; + } + } + + if (hmx_eligible) { + size_t Br = 0, Bc = 0; + const size_t vtcm_budget = ctx->socinfo.vtcm_size_in_mb * 1024 * 1024; + int ret = hmx_fa_find_chunk_size(&Br, &Bc, G, DK, DV, neq1, nek1, + vtcm_budget, (size_t) ctx->n_threads, + kparams->is_q_fp32 != 0); + if (ret == 0) { + kparams->kernel_type = HTP_FA_KERNEL_HMX; + kparams->Br = (uint16_t) Br; + kparams->Bc = (uint16_t) Bc; + kparams->n_kv_blocks = (uint16_t)((nek1 + Bc - 1) / Bc); + kparams->n_threads = (kparams->n_kv_blocks >= 3 && ctx->n_threads >= 2) + ? (uint8_t) ctx->n_threads : 1; + kparams->u.hmx.g_br = hex_align_up(G * Br, 32); + kparams->u.hmx.pipeline = (kparams->n_kv_blocks >= 3 && ctx->n_threads >= 2) ? 1 : 0; + kparams->vtcm_size = (uint32_t) hmx_fa_compute_vtcm_usage( + G, DK, DV, Br, Bc, kparams->n_threads, kparams->u.hmx.pipeline != 0, + kparams->is_q_fp32 != 0); + + const size_t row_vec_bytes = hex_align_up(Bc * sizeof(uint16_t), 256); + kparams->u.hmx.row_buf_stride = row_vec_bytes / 128; + const size_t m_line_bytes = hex_align_up(Bc * sizeof(uint16_t), 128); + kparams->u.hmx.mask_buf_row_stride = m_line_bytes / sizeof(uint16_t); + kparams->u.hmx.mask_broadcast = (mask && mask->ne[2] == 1) ? 1 : 0; + kparams->u.hmx.div_G = init_fastdiv_values(G); + if (mask) { + kparams->src3_div2 = init_fastdiv_values((uint32_t) mask->ne[2]); + kparams->src3_div3 = init_fastdiv_values((uint32_t) mask->ne[3]); + } + kparams->qrows = 0; + kparams->qrows_per_thread = 0; + return true; + } + } + + // Fallback to HVX + kparams->kernel_type = HTP_FA_KERNEL_HVX; + kparams->Br = 1; + kparams->Bc = 64; + kparams->n_kv_blocks = (uint16_t)((k->ne[1] + 64 - 1) / 64); + kparams->n_threads = (uint8_t) ctx->n_threads; + kparams->vtcm_size = (uint32_t) hvx_fa_compute_vtcm_usage( + DK, DV, kparams->is_q_fp32 != 0, mask != nullptr, (size_t) ctx->n_threads); + + kparams->u.hvx.size_q_row_padded = hex_round_up((uint32_t)(q->ne[0] * (kparams->is_q_fp32 ? 4 : 2)), 128); + kparams->u.hvx.size_k_row_padded = hex_round_up((uint32_t)(k->ne[0] * 2), 128); + kparams->u.hvx.size_v_row_padded = hex_round_up((uint32_t)(v->ne[0] * 2), 128); + kparams->u.hvx.src0_div21 = init_fastdiv_values((uint32_t)(q->ne[2] * q->ne[1])); + kparams->u.hvx.src0_div1 = init_fastdiv_values((uint32_t) q->ne[1]); + kparams->broadcast_rk2 = init_fastdiv_values((uint32_t)(q->ne[2] / k->ne[2])); + kparams->broadcast_rk3 = init_fastdiv_values((uint32_t)(q->ne[3] / k->ne[3])); + kparams->broadcast_rv2 = init_fastdiv_values((uint32_t)(q->ne[2] / v->ne[2])); + kparams->broadcast_rv3 = init_fastdiv_values((uint32_t)(q->ne[3] / v->ne[3])); + if (mask) { + kparams->src3_div2 = init_fastdiv_values((uint32_t) mask->ne[2]); + kparams->src3_div3 = init_fastdiv_values((uint32_t) mask->ne[3]); + } + kparams->qrows = (uint32_t)(q->ne[1] * q->ne[2] * q->ne[3]); + kparams->qrows_per_thread = (kparams->qrows + ctx->n_threads - 1) / ctx->n_threads; + return true; +} + +// Map GGML opcode to HTP opcode for unary-family ops. Mirrors the DSP-side +// ggml_op_to_htp_op() in htp/entry.c, restricted to the subset that +// htp_op_is_unary() in unary-ops.h accepts. +// Returns false if the op is not a precompute-required unary. +static bool ggml_op_to_htp_op_unary(int32_t ggml_op, const int32_t * op_params, uint32_t * htp_op) { + switch (ggml_op) { + case GGML_OP_NORM: *htp_op = HTP_OP_NORM; return true; + case GGML_OP_L2_NORM: *htp_op = HTP_OP_L2_NORM; return true; + case GGML_OP_RMS_NORM:*htp_op = HTP_OP_RMS_NORM; return true; + case GGML_OP_SCALE: *htp_op = HTP_OP_SCALE; return true; + case GGML_OP_SQR: *htp_op = HTP_OP_SQR; return true; + case GGML_OP_SQRT: *htp_op = HTP_OP_SQRT; return true; + case GGML_OP_TRI: *htp_op = HTP_OP_TRI; return true; + case GGML_OP_UNARY: + if (!op_params) return false; + switch (op_params[0]) { + case GGML_UNARY_OP_NEG: *htp_op = HTP_OP_UNARY_NEG; return true; + case GGML_UNARY_OP_TANH: *htp_op = HTP_OP_UNARY_TANH; return true; + case GGML_UNARY_OP_SIGMOID: *htp_op = HTP_OP_UNARY_SIGMOID; return true; + case GGML_UNARY_OP_EXP: *htp_op = HTP_OP_UNARY_EXP; return true; + case GGML_UNARY_OP_SOFTPLUS: *htp_op = HTP_OP_UNARY_SOFTPLUS; return true; + case GGML_UNARY_OP_SILU: *htp_op = HTP_OP_UNARY_SILU; return true; + case GGML_UNARY_OP_GELU: + case GGML_UNARY_OP_GELU_QUICK: *htp_op = HTP_OP_UNARY_GELU; return true; + default: return false; + } + default: + return false; + } +} + +// Precompute htp_unary_kernel_params on AP side for unary ops (NORM, RMS_NORM, +// RMS_NORM_MUL, SCALE, SQR, SQRT, UNARY_*, L2_NORM, TRI). +static void ggml_hexagon_precompute_unary_params( + const ggml_backend_hexagon_context * ctx, + uint32_t op, + const ggml_tensor * src0, + const ggml_tensor * src1, + const ggml_tensor * dst, + struct htp_unary_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const uint32_t src0_nrows = (uint32_t)(src0->ne[1] * src0->ne[2] * src0->ne[3]); + const uint32_t n_threads = (ctx->n_threads < (int) src0_nrows) ? (uint32_t) ctx->n_threads : src0_nrows; + + kparams->n_threads = n_threads; + + const size_t src0_data_row_size = (size_t) src0->ne[0] * sizeof(float); + const size_t dst_data_row_size = (size_t) dst->ne[0] * sizeof(float); + + const size_t src0_row_size_aligned = hex_round_up((uint32_t) src0_data_row_size, 128); + const size_t dst_row_size_aligned = hex_round_up((uint32_t) dst_data_row_size, 128); + + kparams->src0_row_size_aligned = (uint32_t) src0_row_size_aligned; + kparams->dst_row_size_aligned = (uint32_t) dst_row_size_aligned; + + size_t src1_data_row_size = 0; + size_t src1_row_size_aligned = 0; + bool broadcast_weight = false; + + if (op == HTP_OP_RMS_NORM_MUL) { + GGML_ASSERT(src1 != nullptr); + src1_data_row_size = (size_t) src1->ne[0] * sizeof(float); + src1_row_size_aligned = hex_round_up((uint32_t) src1_data_row_size, 128); + broadcast_weight = (src1->ne[1] * src1->ne[2] * src1->ne[3] == 1); + } + + kparams->src1_row_size_aligned = (uint32_t) src1_row_size_aligned; + kparams->broadcast_weight = broadcast_weight ? 1u : 0u; + + const size_t vtcm_size_budget = ctx->socinfo.vtcm_size_in_mb * 1024ull * 1024ull; + + struct htp_unary_vtcm_layout L; + uint32_t col_tile = 0; + uint32_t vtcm_row_per_thread = 0; + + htp_unary_vtcm_layout_build(&L, op, (uint32_t) src0->ne[0], (uint32_t) dst->ne[0], + op == HTP_OP_RMS_NORM_MUL ? (uint32_t) src1->ne[0] : 0, + broadcast_weight, n_threads, vtcm_size_budget, + &col_tile, &vtcm_row_per_thread); + + kparams->col_tile = col_tile; + kparams->vtcm_row_per_thread = vtcm_row_per_thread; + kparams->vtcm_size = (uint32_t) L.total_bytes; + + kparams->vtcm_src0_size_per_thread = (uint32_t) L.src0_bytes; + kparams->vtcm_src1_size_per_thread = (uint32_t) L.src1_bytes; + kparams->vtcm_dst_size_per_thread = (uint32_t) L.dst_bytes; + + kparams->vtcm_src0_size = (uint32_t)(L.src0_bytes * n_threads); + kparams->vtcm_src1_size = (uint32_t)(L.src1_bytes * n_threads); + kparams->vtcm_dst_size = (uint32_t)(L.dst_bytes * n_threads); + + kparams->block = col_tile ? 0u : (uint32_t) ((L.src0_bytes / 2) / src0_row_size_aligned); + + const uint32_t tiles_per_row = col_tile > 0 ? ((uint32_t) src0->ne[0] + col_tile - 1) / col_tile : 1u; + kparams->div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]); + kparams->div_ne02 = init_fastdiv_values((uint32_t) src0->ne[2]); + kparams->div_ne012 = init_fastdiv_values((uint32_t)(src0->ne[1] * src0->ne[2])); + kparams->div_tpr = init_fastdiv_values(tiles_per_row); +} + +static bool ggml_hexagon_matmul_is_hmx_eligible( + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + int ne01_padded, + bool is_matmul_id, + bool is_batched +) { + const int ne00 = src0->ne[0]; + const int ne11 = src1->ne[1]; + const int ne12 = src1->ne[2]; + // weights may be stored in a different format (see set_tensor) + const int wtype = ggml_hexagon_weight_dsp_type(src0->type); + + if (ne01_padded % 32 != 0) { + return false; + } + + if (!ggml_hexagon_is_hmx_weight_type((ggml_type) wtype)) { + return false; + } + + if (ne00 % 32 != 0) { + return false; + } + + if (!is_matmul_id && is_batched && wtype != GGML_TYPE_F16) { + return false; + } + + if (src0->nb[0] > src0->nb[1] || src1->nb[0] > src1->nb[1]) { + return false; + } + + const int m = is_matmul_id ? ne12 : ne11; + if (m <= HTP_MM_HMX_MIN_NROWS) { + return false; + } + + return true; +} + +// Shared HMX eligibility check: computes standard params from src0/src1/dst +// and delegates to ggml_hexagon_matmul_is_hmx_eligible. Used by +// mm_is_hmx_eligible (opfusion gate) to decide QKV/FFN merge eligibility. +static bool ggml_hexagon_mm_is_hmx_eligible_shared( + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst +) { + const int wtype = src0->type; + const bool is_repack = ggml_hexagon_is_repack_type((ggml_type) wtype); + const bool is_matmul_id = (dst->op == GGML_OP_MUL_MAT_ID); + const bool is_batched = (src0->ne[2] * src0->ne[3] > 1 || src1->ne[2] * src1->ne[3] > 1); + const int ne01_padded = is_repack ? (int) hex_round_up((uint32_t) src0->ne[1], 32) : src0->ne[1]; + + return ggml_hexagon_matmul_is_hmx_eligible(src0, src1, dst, ne01_padded, is_matmul_id, is_batched); +} + +// Gate for QKV/FFN fusion eligibility: returns true when the MUL_MAT +// is suitable for HMX. Consulted only by is_mergeable_mul_mat to avoid +// merging MUL_MATs that would benefit from HMX. +// HMX dispatch itself is decided independently by ggml_hexagon_precompute_mm_params. +static bool mm_is_hmx_eligible(const ggml_backend_hexagon_context * ctx, const ggml_tensor * t) { + if (!ctx->has_hmx) return false; + return ggml_hexagon_mm_is_hmx_eligible_shared(t->src[0], t->src[1], t); +} + +// A MUL_MAT is fusion-eligible when: +// - src0 is quantized (Q4_0/Q8_0/etc.) +// - src1 is F32 (fusion kernels read F32 activations) +// - !mm_is_hmx_eligible (avoid merging MUL_MATs that would benefit from HMX) +// NOTE: fusion is only attempted for MUL_MATs that match the QKV/FFN pattern. +static bool is_mergeable_mul_mat(const ggml_backend_hexagon_context * ctx, const ggml_tensor * t) { + if (!t || t->op != GGML_OP_MUL_MAT) return false; + if (t->src[1]->type != GGML_TYPE_F32) return false; + return ggml_is_quantized(t->src[0]->type) && !mm_is_hmx_eligible(ctx, t); +} + +static bool is_mergeable_mul_mat_pair(const ggml_backend_hexagon_context * ctx, const ggml_tensor * n1, const ggml_tensor * n2) { + if (!is_mergeable_mul_mat(ctx, n1) || !is_mergeable_mul_mat(ctx, n2)) { + return false; + } + if (n1->src[1] != n2->src[1]) { + return false; + } + if (n1->src[0]->ne[0] != n2->src[0]->ne[0] || + n1->src[0]->ne[1] != n2->src[0]->ne[1]) { + return false; + } + if (n1->src[0]->type != n2->src[0]->type) { + return false; + } + return true; +} + +static bool is_qkv_mergeable(const ggml_backend_hexagon_context * ctx, const ggml_tensor * n_q, const ggml_tensor * n_k, const ggml_tensor * n_v) { + if (!is_mergeable_mul_mat(ctx, n_q) || !is_mergeable_mul_mat(ctx, n_k) || !is_mergeable_mul_mat(ctx, n_v)) { + return false; + } + if (n_q->src[1] != n_k->src[1] || n_q->src[1] != n_v->src[1]) { + return false; + } + if (n_q->src[0]->type != n_k->src[0]->type || n_q->src[0]->type != n_v->src[0]->type) { + return false; + } + if (n_k->src[0]->ne[0] != n_v->src[0]->ne[0] || + n_k->src[0]->ne[1] != n_v->src[0]->ne[1]) { + return false; + } + if (n_q->src[0]->ne[0] != n_k->src[0]->ne[0]) { + return false; + } + // Verify all three outputs share the same sequence/batch dimension (ne[1]) + if (n_q->ne[1] != n_k->ne[1] || n_q->ne[1] != n_v->ne[1]) { + return false; + } + return true; +} + +// Precompute htp_mm_kernel_params for fused QKV matmul (3 outputs: K, V, Q). +// src0 = Wk (representative of K/V/Q weights), src1 = x (shared activation). +// DSP-side op_matmul_qkv expects src[0]=Wk, src[1]=x, src[2]=Wv, src[3]=Wq. +static void ggml_hexagon_precompute_fused_qkv_params( + const ggml_backend_hexagon_context * ctx, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + struct htp_mm_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const int wtype = src0->type; + const bool is_repack = ggml_hexagon_is_repack_type((ggml_type) wtype); + + const int ne10 = src1->ne[0]; + const int src1_nrows = src1->ne[1] * src1->ne[2] * src1->ne[3]; + const size_t src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); + const size_t src0_row_size = src0->nb[1]; + const size_t src0_row_size_padded = hex_round_up((uint32_t) src0_row_size, 128); + + size_t src0_sz_per_thread = 0; + size_t src2_sz_per_thread = 0; + size_t src3_sz_per_thread = 0; + uint32_t best_n_prefetch = 16; + + const size_t vtcm_budget = ctx->socinfo.vtcm_size_in_mb * 1024 * 1024; + size_t quant_scratch_size = hex_round_up((uint32_t)(ne10 * sizeof(float)), QK_Q8_0_TILED * sizeof(float)) * (uint32_t)ctx->n_threads; + + if (is_repack) { + uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype); + uint32_t n_k_tiles = hex_round_up((uint32_t) ne10, 32) / 32; + uint32_t tile_row_size = n_k_tiles * aligned_tile_size; + size_t src1_sz_per_thread = hex_round_up((uint32_t)(src1_row_size * src1_nrows), 128); + size_t src1_sz = src1_sz_per_thread; + + const uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16; + best_n_prefetch = 2; + for (uint32_t d = max_prefetch; d >= 2; d /= 2) { + size_t repacked_vtcm_size = hex_round_up(d * tile_row_size, 128); + size_t src0_sz = repacked_vtcm_size * (uint32_t)ctx->n_threads; + size_t src2_sz = hex_round_up(d * tile_row_size, 128) * (uint32_t)ctx->n_threads; + size_t src3_sz = hex_round_up(d * tile_row_size, 128) * (uint32_t)ctx->n_threads; + size_t tiled_vtcm_size = src0_sz + src1_sz + src2_sz + src3_sz + quant_scratch_size; + + if (tiled_vtcm_size <= vtcm_budget) { + best_n_prefetch = d; + src0_sz_per_thread = repacked_vtcm_size; + src2_sz_per_thread = hex_round_up(d * tile_row_size, 128); + src3_sz_per_thread = hex_round_up(d * tile_row_size, 128); + break; + } + } + if (best_n_prefetch == 2 && src0_sz_per_thread == 0) { + size_t repacked_vtcm_size = hex_round_up(2 * tile_row_size, 128); + src0_sz_per_thread = repacked_vtcm_size; + src2_sz_per_thread = hex_round_up(2 * tile_row_size, 128); + src3_sz_per_thread = hex_round_up(2 * tile_row_size, 128); + } + } else { + best_n_prefetch = 16; + src0_sz_per_thread = hex_round_up((uint32_t)(best_n_prefetch * src0_row_size_padded), 128); + src2_sz_per_thread = hex_round_up((uint32_t)(best_n_prefetch * src0_row_size_padded), 128); + src3_sz_per_thread = hex_round_up((uint32_t)(best_n_prefetch * src0_row_size_padded), 128); + } + + size_t src1_sz_per_thread = hex_round_up((uint32_t)(src1_row_size * src1_nrows), 128); + + size_t src0_sz = src0_sz_per_thread * (uint32_t)ctx->n_threads; + size_t src1_sz = src1_sz_per_thread; + size_t src2_sz = src2_sz_per_thread * (uint32_t)ctx->n_threads; + size_t src3_sz = src3_sz_per_thread * (uint32_t)ctx->n_threads; + + size_t tiled_vtcm_size = src0_sz + src1_sz + src2_sz + src3_sz + quant_scratch_size; + bool try_tiled = true; + if (try_tiled && tiled_vtcm_size <= vtcm_budget) { + kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW; + kparams->vtcm_src0_size = (int32_t) src0_sz; + kparams->vtcm_src1_size = (int32_t) src1_sz; + kparams->vtcm_src2_size = (int32_t) src2_sz; + kparams->vtcm_src3_size = (int32_t) src3_sz; + kparams->vtcm_dst_size = (int32_t) quant_scratch_size; + kparams->vtcm_size = (int32_t) tiled_vtcm_size; + kparams->n_prefetch = (int32_t) best_n_prefetch; + } else { + kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT; + size_t flat_src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10); + size_t flat_src1_sz = hex_round_up((uint32_t)(flat_src1_row_size * src1_nrows), 128); + kparams->vtcm_src0_size = (int32_t) src0_sz; + kparams->vtcm_src1_size = (int32_t) flat_src1_sz; + kparams->vtcm_src2_size = (int32_t) src2_sz; + kparams->vtcm_src3_size = (int32_t) src3_sz; + kparams->vtcm_dst_size = (int32_t) quant_scratch_size; + kparams->vtcm_size = (int32_t)(src0_sz + flat_src1_sz + src2_sz + src3_sz + quant_scratch_size); + kparams->n_prefetch = (int32_t) best_n_prefetch; + } +} + +// Precompute htp_mm_kernel_params for fused FFN matmul (2 outputs: gate, up). +// src0 = Wgate, src1 = y (shared activation). +// DSP-side op_matmul_ffn expects src[0]=Wgate, src[1]=y, src[2]=Wup. +static void ggml_hexagon_precompute_fused_ffn_params( + const ggml_backend_hexagon_context * ctx, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + struct htp_mm_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const int wtype = src0->type; + const bool is_repack = ggml_hexagon_is_repack_type((ggml_type) wtype); + + const int ne10 = src1->ne[0]; + const int src1_nrows = src1->ne[1] * src1->ne[2] * src1->ne[3]; + const size_t src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10); + const size_t src0_row_size = src0->nb[1]; + const size_t src0_row_size_padded = hex_round_up((uint32_t) src0_row_size, 128); + + size_t src0_sz_per_thread = 0; + size_t src2_sz_per_thread = 0; + uint32_t best_n_prefetch = 16; + + const size_t vtcm_budget = ctx->socinfo.vtcm_size_in_mb * 1024 * 1024; + size_t quant_scratch_size = hex_round_up((uint32_t)(ne10 * sizeof(float)), QK_Q8_0_TILED * sizeof(float)) * (uint32_t)ctx->n_threads; + + if (is_repack) { + uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype); + uint32_t n_k_tiles = hex_round_up((uint32_t) ne10, 32) / 32; + uint32_t tile_row_size = n_k_tiles * aligned_tile_size; + size_t src1_sz_per_thread = hex_round_up((uint32_t)(src1_row_size * src1_nrows), 128); + size_t src1_sz = src1_sz_per_thread; + + const uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16; + best_n_prefetch = 2; + for (uint32_t d = max_prefetch; d >= 2; d /= 2) { + size_t repacked_vtcm_size = hex_round_up(d * tile_row_size, 128); + size_t src0_sz = repacked_vtcm_size * (uint32_t)ctx->n_threads; + size_t src2_sz = hex_round_up(d * tile_row_size, 128) * (uint32_t)ctx->n_threads; + size_t tiled_vtcm_size = src0_sz + src1_sz + src2_sz + quant_scratch_size; + + if (tiled_vtcm_size <= vtcm_budget) { + best_n_prefetch = d; + src0_sz_per_thread = repacked_vtcm_size; + src2_sz_per_thread = hex_round_up(d * tile_row_size, 128); + break; + } + } + if (best_n_prefetch == 2 && src0_sz_per_thread == 0) { + size_t repacked_vtcm_size = hex_round_up(2 * tile_row_size, 128); + src0_sz_per_thread = repacked_vtcm_size; + src2_sz_per_thread = hex_round_up(2 * tile_row_size, 128); + } + } else { + best_n_prefetch = 16; + src0_sz_per_thread = hex_round_up((uint32_t)(best_n_prefetch * src0_row_size_padded), 128); + src2_sz_per_thread = hex_round_up((uint32_t)(best_n_prefetch * src0_row_size_padded), 128); + } + + size_t src1_sz_per_thread = hex_round_up((uint32_t)(src1_row_size * src1_nrows), 128); + + size_t src0_sz = src0_sz_per_thread * (uint32_t)ctx->n_threads; + size_t src1_sz = src1_sz_per_thread; + size_t src2_sz = src2_sz_per_thread * (uint32_t)ctx->n_threads; + + size_t tiled_vtcm_size = src0_sz + src1_sz + src2_sz + quant_scratch_size; + bool try_tiled = true; + if (try_tiled && tiled_vtcm_size <= vtcm_budget) { + kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW; + kparams->vtcm_src0_size = (int32_t) src0_sz; + kparams->vtcm_src1_size = (int32_t) src1_sz; + kparams->vtcm_src2_size = (int32_t) src2_sz; + kparams->vtcm_dst_size = (int32_t) quant_scratch_size; + kparams->vtcm_size = (int32_t) tiled_vtcm_size; + kparams->n_prefetch = (int32_t) best_n_prefetch; + } else { + kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT; + size_t flat_src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_flat_row_size(ne10) : htp_mm_q8_0_flat_row_size(ne10); + size_t flat_src1_sz = hex_round_up((uint32_t)(flat_src1_row_size * src1_nrows), 128); + kparams->vtcm_src0_size = (int32_t) src0_sz; + kparams->vtcm_src1_size = (int32_t) flat_src1_sz; + kparams->vtcm_src2_size = (int32_t) src2_sz; + kparams->vtcm_dst_size = (int32_t) quant_scratch_size; + kparams->vtcm_size = (int32_t)(src0_sz + flat_src1_sz + src2_sz + quant_scratch_size); + kparams->n_prefetch = (int32_t) best_n_prefetch; + } +} + +// Precompute htp_mm_kernel_params on AP side for MUL_MAT in FastRPC/mempool batch path. +// Mirrors build_mm_kernel_params in htp/entry.c (F32/F16 HVX paths only). +// Writes directly to op.kernel_params; DSP side consumes via memcpy. +// For unsupported weight types (quant/HMX), leaves kernel_type=0 so DSP falls +// back to build_mm_kernel_params which emits the error. +// When is_matmul_id=false, the node is a plain MUL_MAT (not MUL_MAT_ID). +// The HMX-first-then-HVX-fallback policy is preserved. +static bool ggml_hexagon_precompute_hmx_mm_params( + const ggml_backend_hexagon_context * ctx, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + int wtype, + int ne00_padded, + int ne01_padded, + int ne02, + int ne11, + int ne12, + int ne11_padded, + bool is_matmul_id, + bool is_batched, + size_t vtcm_budget, + struct htp_mm_kernel_params * kparams +) { + const int aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype); + // Force the pipelined path for plain MUL_MAT (MUL_MAT_ID keeps the + // non-pipelined setting, matching upstream): the synchronous (m<=32) + // branch yields corrupted, non-deterministic output in this integration + // (observed with ubatch<=32 for Qwen3.5-2B-Q4_0.gguf) + // Qualcomm uses the following logic to decide whether to enable HMX pipeline: + // const bool pipeline = is_matmul_id ? false : htp_mm_hmx_pipeline(ne11); + const bool pipeline = is_matmul_id ? false : true; + const int n_threads = (int) ctx->n_threads; + const int ne10 = src1->ne[0]; + + const bool is_batched_val = is_matmul_id ? false : is_batched; + const int group_size = (ne02 > 0 ? ne12 / ne02 : 1); + + size_t m_chunk = 0; + size_t n_chunk = 0; + size_t vtcm_size = 0; + bool use_grouped = false; + int act_threads_selected = 0; + + GGMLHEXAGON_LOG_DEBUG("ne00 %d, ne01 %d, ne02 %d, ne10 %d, ne11 %d, ne12 %d", src0->ne[0], src0->ne[1], src0->ne[2], src1->ne[0], src1->ne[1], src1->ne[2]); + + if (is_batched_val && wtype == GGML_TYPE_F16 && group_size > 1) { + // Try grouped path first + const bool use_dma_activation = (src1->nb[1]/sizeof(float) > (size_t)ne00_padded); + if (htp_mm_hmx_solve_batched_params(wtype, ne00_padded, ne01_padded, ne11, + group_size, use_dma_activation, n_threads, pipeline, + vtcm_budget, &m_chunk, &n_chunk, + &act_threads_selected, &vtcm_size)) { + use_grouped = true; + } + } + + if (!use_grouped) { + // Fallback to simple 2D path (group_size = 1) + const int m_id_rows = (int) ((size_t) dst->ne[1] * dst->ne[2]); + if (!htp_mm_hmx_solve_2d_params(wtype, ne00_padded, m_id_rows, + ne01_padded, ne11_padded, ne11, n_threads, pipeline, + is_matmul_id, aligned_tile_size, vtcm_budget, + &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) { + return false; + } + } + + kparams->n_hmx = 1; + kparams->pipeline = pipeline ? 1 : 0; + kparams->m_chunk = (int32_t) m_chunk; + kparams->n_chunk = (int32_t) n_chunk; + kparams->n_threads = n_threads; + kparams->n_act_threads = act_threads_selected; + kparams->tile_size = (int32_t) htp_mm_get_weight_tile_size(wtype); + kparams->aligned_tile_size = (int32_t) aligned_tile_size; + kparams->src1_row_size = (int32_t)((wtype == GGML_TYPE_Q4_1) + ? htp_mm_q8_1_tiled_row_size(ne10) + : htp_mm_q8_0_tiled_row_size(ne10)); + kparams->vtcm_size = (int32_t) vtcm_size; + kparams->vtcm_src0_size = 0; + kparams->div_n_act_threads = init_fastdiv_values((uint32_t) act_threads_selected); + kparams->div_ne00_padded = init_fastdiv_values((uint32_t) ne00_padded); + kparams->vtcm_src1_size = 0; + kparams->vtcm_dst_size = 0; + + if (is_batched && !is_matmul_id) { + kparams->kernel_type = HTP_MM_KERNEL_HMX_F16_BATCHED; + } else { + kparams->kernel_type = HTP_MM_KERNEL_HMX_2D; + } + GGMLHEXAGON_LOG_DEBUG("wtype=%d k=%lld n=%lld m=%lld pip=%d mc=%d nc=%d act=%d vtcm=%d nt=%d", + (int) wtype, (long long) src0->ne[0], (long long) src0->ne[1], (long long) ne11, + (int) pipeline, kparams->m_chunk, kparams->n_chunk, kparams->n_act_threads, + kparams->vtcm_size, ctx->n_threads); + + return true; +} + +static void ggml_hexagon_precompute_hvx_mm_params( + const ggml_backend_hexagon_context * ctx, + const struct ggml_tensor * src0, + const struct ggml_tensor * src1, + const struct ggml_tensor * dst, + int wtype, + int ne02, + int ne03, + int ne10, + int ne11, + int ne12, + int ne13, + bool is_matmul_id, + size_t vtcm_budget, + struct htp_mm_kernel_params * kparams +) { + kparams->n_hmx = 0; + + const bool is_quant = (wtype != GGML_TYPE_F16 && wtype != GGML_TYPE_F32); + const int src1_nrows = ne11 * ne12 * ne13; + + if (is_quant) { + // Quantized HVX + kparams->tile_size = (int32_t) htp_mm_get_weight_tile_size(wtype); + kparams->aligned_tile_size = (int32_t) htp_mm_get_weight_aligned_tile_size(wtype); + + const bool k_align = (ne10 % 32 == 0); + + if (is_matmul_id) { + kparams->kernel_type = (src1_nrows < (int) ctx->n_threads) + ? HTP_MM_KERNEL_HVX_QUANT_BLOCK : HTP_MM_KERNEL_HVX_QUANT_ROW; + kparams->src1_row_size = (int32_t)((wtype == GGML_TYPE_Q4_1) + ? htp_mm_q8_1_tiled_row_size(ne10) + : htp_mm_q8_0_tiled_row_size(ne10)); + + size_t vtcm_src0_size = 0, vtcm_src1_size = 0, vtcm_dst_size = 0; + uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16; + uint32_t best_n_prefetch = 2; + size_t total_size = 0; + for (uint32_t d = max_prefetch; d >= 2; d /= 2) { + total_size = htp_mm_hvx_id_get_vtcm_sizes( + wtype, ne10, src1_nrows, (uint32_t)ctx->n_threads, + src0->nb[1], d, + &vtcm_src0_size, &vtcm_src1_size, &vtcm_dst_size + ); + if (total_size <= vtcm_budget) { + best_n_prefetch = d; + break; + } + } + if (best_n_prefetch == 2 && total_size > vtcm_budget) { + total_size = htp_mm_hvx_id_get_vtcm_sizes( + wtype, ne10, src1_nrows, (uint32_t)ctx->n_threads, + src0->nb[1], 2, + &vtcm_src0_size, &vtcm_src1_size, &vtcm_dst_size + ); + } + kparams->n_prefetch = (int32_t) best_n_prefetch; + kparams->vtcm_size = (int32_t) total_size; + kparams->vtcm_src0_size = (int32_t) vtcm_src0_size; + kparams->vtcm_src1_size = (int32_t) vtcm_src1_size; + kparams->vtcm_dst_size = (int32_t) vtcm_dst_size; + } else { + if (k_align) { + kparams->src1_row_size = (int32_t)((wtype == GGML_TYPE_Q4_1) + ? htp_mm_q8_1_tiled_row_size(ne10) + : htp_mm_q8_0_tiled_row_size(ne10)); + if (src1_nrows < (int)ctx->n_threads) { + kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_BLOCK; + } else { + kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW; + } + + uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16; + uint32_t best_n_prefetch = 2; + size_t vtcm_src0_size = 0, vtcm_src1_size = 0, vtcm_dst_size = 0; + size_t total_size = 0; + for (uint32_t d = max_prefetch; d >= 2; d /= 2) { + total_size = htp_mm_hvx_get_vtcm_sizes( + kparams->kernel_type, wtype, ne10, src1_nrows, + (uint32_t)ctx->n_threads, + dst->nb[1], src0->nb[1], src1->nb[1], d, + &vtcm_src0_size, &vtcm_src1_size, &vtcm_dst_size + ); + if (total_size <= vtcm_budget) { + best_n_prefetch = d; + break; + } + } + if (best_n_prefetch == 2 && total_size > vtcm_budget) { + total_size = htp_mm_hvx_get_vtcm_sizes( + kparams->kernel_type, wtype, ne10, src1_nrows, + (uint32_t)ctx->n_threads, + dst->nb[1], src0->nb[1], src1->nb[1], 2, + &vtcm_src0_size, &vtcm_src1_size, &vtcm_dst_size + ); + } + + kparams->n_prefetch = (int32_t) best_n_prefetch; + + if (total_size <= vtcm_budget) { + kparams->vtcm_size = (int32_t) total_size; + kparams->vtcm_src0_size = (int32_t) vtcm_src0_size; + kparams->vtcm_src1_size = (int32_t) vtcm_src1_size; + kparams->vtcm_dst_size = (int32_t) vtcm_dst_size; + goto done_quant; + } + GGMLHEXAGON_LOG_DEBUG("precompute_hvx: tiled path VTCM too large " + "(need=%zu budget=%zu), falling back to flat", + total_size, vtcm_budget); + } + + // Flat HVX fallback + { + kparams->src1_row_size = (int32_t)((wtype == GGML_TYPE_Q4_1) + ? htp_mm_q8_1_flat_row_size(ne10) + : htp_mm_q8_0_flat_row_size(ne10)); + kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT; + + size_t vtcm_src0_size = 0, vtcm_src1_size = 0, vtcm_dst_size = 0; + size_t total_size = htp_mm_hvx_get_vtcm_sizes( + kparams->kernel_type, wtype, ne10, src1_nrows, + (uint32_t)ctx->n_threads, + dst->nb[1], src0->nb[1], src1->nb[1], 16, + &vtcm_src0_size, &vtcm_src1_size, &vtcm_dst_size + ); + + kparams->n_prefetch = 16; + kparams->vtcm_size = (int32_t) total_size; + kparams->vtcm_src0_size = (int32_t) vtcm_src0_size; + kparams->vtcm_src1_size = (int32_t) vtcm_src1_size; + kparams->vtcm_dst_size = (int32_t) vtcm_dst_size; + } + } + + done_quant:; + } else if (wtype == GGML_TYPE_F16) { + // F16 HVX + const bool is_batched = (ne02 > 1) || (ne03 > 1); + const bool is_permuted = ggml_is_permuted(src0) || ggml_is_permuted(src1); + + size_t vtcm_src0_size = 0, vtcm_src1_size = 0, vtcm_dst_size = 0; + size_t vtcm_size = htp_mm_hvx_get_vtcm_sizes( + HTP_MM_KERNEL_HVX_F16_F16_VTCM, wtype, ne10, src1_nrows, + (uint32_t)ctx->n_threads, + dst->nb[1], src0->nb[1], src1->nb[1], 16, + &vtcm_src0_size, &vtcm_src1_size, &vtcm_dst_size + ); + + if (!is_batched && !is_permuted && vtcm_size <= vtcm_budget) { + kparams->kernel_type = HTP_MM_KERNEL_HVX_F16_F16_VTCM; + kparams->src1_row_size = (int32_t) hex_round_up(ne10 * 2, 128); + kparams->vtcm_size = (int32_t) vtcm_size; + kparams->vtcm_src0_size = (int32_t) vtcm_src0_size; + kparams->vtcm_src1_size = (int32_t) vtcm_src1_size; + kparams->vtcm_dst_size = (int32_t) vtcm_dst_size; + kparams->n_prefetch = 16; + } else { + if (src1->type == GGML_TYPE_F32) { + kparams->kernel_type = HTP_MM_KERNEL_HVX_F16_F32_DDR; + } else { + kparams->kernel_type = HTP_MM_KERNEL_HVX_F16_F16_DDR; + } + kparams->src1_row_size = (int32_t) src1->nb[1]; + size_t ddr_size = htp_mm_hvx_get_vtcm_sizes( + kparams->kernel_type, wtype, ne10, src1_nrows, + (uint32_t)ctx->n_threads, + dst->nb[1], src0->nb[1], src1->nb[1], 16, + &vtcm_src0_size, &vtcm_src1_size, &vtcm_dst_size + ); + kparams->vtcm_size = (int32_t) ddr_size; + kparams->vtcm_src0_size = (int32_t) vtcm_src0_size; + kparams->vtcm_src1_size = (int32_t) vtcm_src1_size; + kparams->vtcm_dst_size = (int32_t) vtcm_dst_size; + kparams->n_prefetch = 16; + } + } else { + // F32 HVX + const bool is_batched = (ne02 > 1) || (ne03 > 1); + const bool is_permuted = ggml_is_permuted(src0) || ggml_is_permuted(src1); + + size_t vtcm_src0_size = 0, vtcm_src1_size = 0, vtcm_dst_size = 0; + size_t vtcm_size = htp_mm_hvx_get_vtcm_sizes( + HTP_MM_KERNEL_HVX_F32_F32_VTCM, wtype, ne10, src1_nrows, + (uint32_t)ctx->n_threads, + dst->nb[1], src0->nb[1], src1->nb[1], 16, + &vtcm_src0_size, &vtcm_src1_size, &vtcm_dst_size + ); + + if (!is_batched && !is_permuted && vtcm_size <= vtcm_budget) { + kparams->kernel_type = HTP_MM_KERNEL_HVX_F32_F32_VTCM; + kparams->src1_row_size = (int32_t) hex_round_up(ne10 * 4, 128); + kparams->vtcm_size = (int32_t) vtcm_size; + kparams->vtcm_src0_size = (int32_t) vtcm_src0_size; + kparams->vtcm_src1_size = (int32_t) vtcm_src1_size; + kparams->vtcm_dst_size = (int32_t) vtcm_dst_size; + kparams->n_prefetch = 16; + } else { + kparams->kernel_type = HTP_MM_KERNEL_HVX_F32_F32_DDR; + kparams->src1_row_size = (int32_t) src1->nb[1]; + size_t ddr_size = htp_mm_hvx_get_vtcm_sizes( + kparams->kernel_type, wtype, ne10, src1_nrows, + (uint32_t)ctx->n_threads, + dst->nb[1], src0->nb[1], src1->nb[1], 16, + &vtcm_src0_size, &vtcm_src1_size, &vtcm_dst_size + ); + kparams->vtcm_size = (int32_t) ddr_size; + kparams->vtcm_src0_size = (int32_t) vtcm_src0_size; + kparams->vtcm_src1_size = (int32_t) vtcm_src1_size; + kparams->vtcm_dst_size = (int32_t) vtcm_dst_size; + kparams->n_prefetch = 16; + } + } +} + +static void ggml_hexagon_precompute_mm_params( + ggml_backend_hexagon_context * ctx, + const ggml_tensor * node, + hex_op_desc & op, + bool is_matmul_id +) { + const ggml_tensor * src0 = node->src[0]; + const ggml_tensor * src1 = node->src[1]; + const ggml_tensor * dst = node; + + struct htp_mm_kernel_params * kparams = + (struct htp_mm_kernel_params *) op.kernel_params; + memset(kparams, 0, sizeof(*kparams)); + + const int ne00 = src0->ne[0]; + const int ne01 = src0->ne[1]; + const int ne02 = src0->ne[2]; + const int ne03 = src0->ne[3]; + + const int ne10 = src1->ne[0]; + const int ne11 = src1->ne[1]; + const int ne12 = src1->ne[2]; + const int ne13 = src1->ne[3]; + + // weights may be stored in a different format (see set_tensor); + // select kernels and VTCM layout by the storage type + const int wtype = ggml_hexagon_weight_dsp_type(src0->type); + const bool is_repack = ggml_hexagon_is_repack_type((ggml_type) wtype); + const int ne00_padded = is_repack ? (int) hex_round_up((uint32_t) ne00, 32) : ne00; + const int ne01_padded = is_repack ? (int) hex_round_up((uint32_t) ne01, 32) : ne01; + const int ne11_padded = (int) hex_round_up((uint32_t) ne11, 32); + + const bool is_batched = (ne02 * ne03 > 1 || ne12 * ne13 > 1); + + const size_t vtcm_budget = (size_t)ctx->socinfo.vtcm_size_in_mb * 1024 * 1024; + + // Cache key: tensor ptr (unique per weight object) ^ data ptr (stable + // mempool offset) ^ ne11 (varies for PP batched matmul, fixed for TG). + // Including src0 tensor ptr avoids mempool-region-reuse collisions: if the + // mempool is recycled across model loads, the same data ptr may point + // to a different weight tensor, but src0 will differ. + const uintptr_t cache_key = (uintptr_t) src0 ^ (uintptr_t) src0->data ^ ((uintptr_t) ne11 << 32); + auto it = ctx->mm_params_cache.find(cache_key); + if (it != ctx->mm_params_cache.end()) { + *kparams = it->second; + return; + } + + // HMX-first policy: try HMX precomputation if eligible, fall back to HVX. + bool hmx_enabled = ctx->has_hmx; + bool hmx_basic = hmx_enabled && ggml_hexagon_matmul_is_hmx_eligible( + src0, src1, dst, ne01_padded, is_matmul_id, is_batched); + + // HMX eligibility diagnostic: categorize why basic check failed + if (!hmx_enabled) { + // HMX not available on this SoC; nothing to count + } else if (hmx_basic) { + ctx->n_hmx_basic_pass++; + } else { + // Diagnose which condition failed (order matches ggml_hexagon_matmul_is_hmx_eligible) + if (ne01_padded % 32 != 0) { + ctx->n_hmx_basic_fail_ne01++; + } else if (!ggml_hexagon_is_hmx_weight_type((ggml_type) wtype)) { + ctx->n_hmx_basic_fail_wtype++; + } else if (ne00 % 32 != 0) { + ctx->n_hmx_basic_fail_ne00++; + } else if (!is_matmul_id && is_batched && wtype != GGML_TYPE_F16) { + ctx->n_hmx_basic_fail_batched++; + } else if (src0->nb[0] > src0->nb[1] || src1->nb[0] > src1->nb[1]) { + ctx->n_hmx_basic_fail_permuted++; + } else { + int m = is_matmul_id ? (int)ne12 : (int)ne11; + if (m <= HTP_MM_HMX_MIN_NROWS) { + ctx->n_hmx_basic_fail_small_n++; + } + } + } + + if (hmx_basic) { + if (ggml_hexagon_precompute_hmx_mm_params( + ctx, src0, src1, dst, wtype, ne00_padded, ne01_padded, + ne02, ne11, ne12, ne11_padded, is_matmul_id, is_batched, + vtcm_budget, kparams)) { + ctx->n_hmx_vtcm_pass++; + goto finalize; + } + ctx->n_hmx_vtcm_fail++; + } + + // Fallback to HVX parameter computation + ggml_hexagon_precompute_hvx_mm_params( + ctx, src0, src1, dst, wtype, + ne02, ne03, ne10, ne11, ne12, ne13, + is_matmul_id, vtcm_budget, kparams); + +finalize: + kparams->div_ne12_ne1 = init_fastdiv_values((uint32_t)(ne12 * ne11)); + kparams->div_ne1 = init_fastdiv_values((uint32_t) ne11); + kparams->div_r2 = init_fastdiv_values(ne02 > 0 ? (uint32_t)(ne12 / ne02) : 1); + kparams->div_r3 = init_fastdiv_values(ne03 > 0 ? (uint32_t)(ne13 / ne03) : 1); + kparams->div_ne11 = init_fastdiv_values((uint32_t) ne11); + + // Cache populated: key includes src0 tensor ptr to avoid mempool-reuse + // collisions. The cached kparams are valid for the session lifetime + // because weights are static (never modified after model load). + ctx->mm_params_cache[cache_key] = *kparams; +} + +// ================================================================================================= +// section-9: backend implementation +// ================================================================================================= +ggml_backend_hexagon_context::ggml_backend_hexagon_context(int dev_id, ggml_backend_dev_t dev) + : device(dev_id), + backend(nullptr), + socinfo(), + n_threads(6), + domain_id(CDSP_DOMAIN_ID), + session_id(0), + ggmlop_handle(0), + rpc_mempool_capacity(0), + rpc_mempool_len(0), + rpc_mempool_usage(0), + rpc_mempool_handle(0), + rpc_mempool(nullptr), + rpc_mempool_dsp_base(nullptr), + weights_dirty(false), + dsp_need_weight_inval_reset(false), + rpc_batch_call_count(0), + cumulative_graph_us(0), + last_graph_end_us(0), + max_nodes_per_graph(0), + min_nodes_per_graph(0), + total_nodes_processed(0), + min_graph_us(0), + max_graph_us(0), + max_graph_n_nodes(0), + max_graph_n_ops(0), + min_n_ops_per_call(0), + max_n_ops_per_call(0), + min_p10_us(0), + max_p10_us(0), + max_layer_idx_seen(0), + min_rpc_overhead_us(0), + max_rpc_overhead_us(0), + sum_rpc_overhead_us(0), + cum_p5_us(0), + cum_p6_us(0), + cum_p8_us(0), + cum_p9_us(0), + cum_p10_us(0), + cum_p10_rpc_setup_us(0), + cum_p10_dsp_exec_us(0), + cum_p10_civac_us(0), + cum_p11_us(0), + cum_p12_us(0), + cum_unaccounted_us(0), + rpc_overhead_min_us(0), + rpc_overhead_max_us(0), + rpc_overhead_sum_us(0), + rpc_overhead_count(0), + n_mul_mat_total_cum(0), + n_hmx_used_cum(0), + n_fused_qkv_cum(0), + n_fused_ffn_cum(0), + n_fused_mm_add_cum(0), + n_hmx_basic_pass(0), + n_hmx_basic_fail_ne01(0), + n_hmx_basic_fail_ne00(0), + n_hmx_basic_fail_wtype(0), + n_hmx_basic_fail_batched(0), + n_hmx_basic_fail_permuted(0), + n_hmx_basic_fail_small_n(0), + n_hmx_vtcm_pass(0), + n_hmx_vtcm_fail(0), + buffer_type{}, + has_vtcm(false), + has_hvx(false), + has_hmx(false), + has_async_fastrpc(false), + has_extended_map(false), + warned_qkv_name(false), + set_tensor_call_count(0) { + snprintf(name, sizeof(name), "HTP%d", dev_id); + snprintf(desc, sizeof(desc), "Qualcomm NPU(CDSP%d)", dev_id); + snprintf(buft_name, sizeof(buft_name), "hexagon-ion-buffer-%s", name); + snprintf(repack_buft_name, sizeof(repack_buft_name), "hexagon-ion-buffer-%s-REPACK", name); + lib[0] = '\0'; + + buffer_type.iface.get_name = ggml_backend_hexagon_buffer_type_name; + buffer_type.iface.alloc_buffer = ggml_backend_hexagon_buffer_type_alloc_buffer; + buffer_type.iface.get_alignment = ggml_backend_hexagon_buffer_type_get_alignment; + buffer_type.iface.get_max_size = ggml_backend_hexagon_buffer_type_get_max_size; + buffer_type.iface.get_alloc_size = ggml_backend_hexagon_buffer_type_get_alloc_size; + buffer_type.iface.is_host = ggml_backend_hexagon_buffer_is_host; + buffer_type.device = dev; + buffer_type.context = this; + + // Repack buffer type: same mempool as buffer_type, but is_host=false + repack_buffer_type.iface.get_name = ggml_backend_hexagon_buffer_type_name; + repack_buffer_type.iface.alloc_buffer = ggml_backend_hexagon_buffer_type_alloc_buffer; + repack_buffer_type.iface.get_alignment = ggml_backend_hexagon_buffer_type_get_alignment; + repack_buffer_type.iface.get_max_size = ggml_backend_hexagon_buffer_type_get_max_size; + repack_buffer_type.iface.get_alloc_size = ggml_backend_hexagon_buffer_type_get_alloc_size; + repack_buffer_type.iface.is_host = ggml_backend_hexagon_repack_buffer_is_host; + repack_buffer_type.device = dev; + repack_buffer_type.context = this; + + // Pre-size the cgraph cache to avoid rehashing during inference. With + // max_load_factor=0.5 and reserve(1024) the bucket array can hold ~1024 + // entries before rehash; observed peak is ~227 (gemma-4/qwen3), so this keeps the + // load low and lookup latency stable. + cgraph_cache.max_load_factor(0.5f); + cgraph_cache.reserve(1024); + + int result = ggmlhexagon_init_dsp(this); + if (0 != result) { + GGMLHEXAGON_LOG_ERROR("init hexagon dsp failure for device %d", dev_id); + throw std::runtime_error("ggml-hexagon: failed to init DSP session"); + } +} + +ggml_backend_hexagon_context::~ggml_backend_hexagon_context() { + ggmlhexagon_deinit_cdsp(this); + ggmlhexagon_print_running_timestamp(NULL); +} + +static bool ggmlhexagon_supported_mul_mat(const struct ggml_tensor * dst, + ggml_backend_hexagon_context * ctx) { + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + + const int64_t m = src0->ne[1]; + const int64_t k = src0->ne[0]; + const int64_t n = src1->ne[1]; + const uint32_t src0_rank = ggml_n_dims(src0); + const uint32_t src1_rank = ggml_n_dims(src1); + GGML_UNUSED(m); + GGML_UNUSED(k); + GGML_UNUSED(n); + GGML_UNUSED(src0_rank); + GGML_UNUSED(src1_rank); + GGMLHEXAGON_LOG_DEBUG("MUL_MAT check: m=%lld, n=%lld, k=%lld, src0_rank=%d, src1_rank=%d", (long long)m, (long long)n, (long long)k, src0_rank, src1_rank); + + if (dst->type != GGML_TYPE_F32) { + return false; + } + + if (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) { + return false; + } + + switch (src0->type) { + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q4_0: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_Q4_1: + case GGML_TYPE_MXFP4: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q6_K: + { + // src0 (weights) must be repacked. In test-backend-ops tensors + // are allocated in the main buffer, so this filters quantized MUL_MAT test cases + if (!src0->buffer || !ggml_backend_buffer_is_hexagon_repack(src0->buffer)) { + return false; + } + + // Repack-buft allocations always land inside the mempool (alloc_buffer + // falls back to a CPU buffer otherwise, rejected by the check above); + // Q5_K below keeps an explicit range check as a defensive invariant. + if (src0->ne[0] % 32) { + return false; + } + + if (src1->ne[2] != 1 || src1->ne[3] != 1) { + return false; // no broadcasting (for now) + } + + // Quantized HVX kernels assume src0 is laid out contiguously in + // row-major (ne[0] is innermost). Non-contiguous views (e.g. k_v + // slices) cause wrong tile offsets -> silent numeric corruption. + if (!ggml_is_contiguous(src0)) { + GGMLHEXAGON_LOG_DEBUG("supported_mul_mat FAIL: src0 not contiguous (nb=[%lld,%lld,%lld,%lld] ne=[%lld,%lld,%lld,%lld])", + (long long)src0->nb[0], (long long)src0->nb[1], (long long)src0->nb[2], (long long)src0->nb[3], + (long long)src0->ne[0], (long long)src0->ne[1], (long long)src0->ne[2], (long long)src0->ne[3]); + return false; + } + + break; + } + + case GGML_TYPE_Q5_K: + { + // src0 (weights) must be repacked. In test-backend-ops tensors + // are allocated in the main buffer, so this filters quantized MUL_MAT test cases + if (!src0->buffer || !ggml_backend_buffer_is_hexagon_repack(src0->buffer)) { + return false; + } + + // Defensive invariant: repack-buft data must sit inside the mempool. + // alloc_buffer never places it on heap (a CPU buffer is returned and + // rejected above), so this should never trigger; kept to catch + // future allocator changes. + if (src0->data) { + const char * dp = (const char *)src0->data; + const char * base = (const char *)ctx->rpc_mempool; + if (dp < base || dp >= base + (ptrdiff_t)ctx->rpc_mempool_len) { + return false; + } + } + + if (src0->ne[0] % 32) { + return false; + } + + if (src1->ne[2] != 1 || src1->ne[3] != 1) { + return false; // no broadcasting (for now) + } + + // Quantized HVX kernels assume src0 is laid out contiguously in + // row-major (ne[0] is innermost). Non-contiguous views (e.g. k_v + // slices) cause wrong tile offsets -> silent numeric corruption. + if (!ggml_is_contiguous(src0)) { + GGMLHEXAGON_LOG_DEBUG("supported_mul_mat FAIL: src0 not contiguous (nb=[%lld,%lld,%lld,%lld] ne=[%lld,%lld,%lld,%lld])", + (long long)src0->nb[0], (long long)src0->nb[1], (long long)src0->nb[2], (long long)src0->nb[3], + (long long)src0->ne[0], (long long)src0->ne[1], (long long)src0->ne[2], (long long)src0->ne[3]); + return false; + } + + break; + } + + case GGML_TYPE_BF16: + // BF16 weights are converted to F16 bytes in the repack buffer at + // load time; only offload when the tensor lives there. + if (!src0->buffer || !ggml_backend_buffer_is_hexagon_repack(src0->buffer)) { + return false; + } + // fall through + case GGML_TYPE_F16: + if (src0->nb[1] < src0->nb[0]) { + GGMLHEXAGON_LOG_WARN("permuted F16 src0 not supported\n"); + return false; + } + if (src1->ne[2] < src0->ne[2] || src1->ne[3] < src0->ne[3]) { + GGMLHEXAGON_LOG_WARN("src1 broadcasting not supported\n"); + return false; + } + break; + + case GGML_TYPE_F32: + if (src1->type != GGML_TYPE_F32) { + return false; + } + if (src0->nb[1] < src0->nb[0]) { + GGMLHEXAGON_LOG_WARN("permuted F32 src0 not supported\n"); + return false; + } + if (src1->ne[2] < src0->ne[2] || src1->ne[3] < src0->ne[3]) { + GGMLHEXAGON_LOG_WARN("src1 broadcasting not supported\n"); + return false; + } + break; + + default: + return false; + } + + // Precompute kernel params to get the actual VTCM size + hex_op_desc tmp_op; + memset(&tmp_op, 0, sizeof(tmp_op)); + tmp_op.opcode = dst->op; + bool is_matmul_id = (dst->op == GGML_OP_MUL_MAT_ID); + ggml_hexagon_precompute_mm_params(ctx, dst, tmp_op, is_matmul_id); + const struct htp_mm_kernel_params * kparams = + (const struct htp_mm_kernel_params *)tmp_op.kernel_params; + + const size_t vtcm_budget = (size_t)ctx->socinfo.vtcm_size_in_mb * 1024 * 1024; + if ((size_t)kparams->vtcm_size > vtcm_budget) { + GGMLHEXAGON_LOG_ALWAYS("MUL_MAT VTCM too small: needed=%d budget=%zu\n", + kparams->vtcm_size, vtcm_budget); + return false; + } + + return true; +} + +static bool ggmlhexagon_supported_flash_attn( + const ggml_backend_hexagon_context * ctx, const struct ggml_tensor * dst) { + const struct ggml_tensor * q = dst->src[0]; + const struct ggml_tensor * k = dst->src[1]; + const struct ggml_tensor * v = dst->src[2]; + const struct ggml_tensor * mask = dst->src[3]; + const struct ggml_tensor * sinks = dst->src[4]; + + if (!q || !k || !v) { + return false; + } + if ((q->type != GGML_TYPE_F16 && q->type != GGML_TYPE_F32) || + k->type != GGML_TYPE_F16 || v->type != GGML_TYPE_F16) { + return false; + } + if (mask && mask->type != GGML_TYPE_F16) { + return false; + } + if (sinks && sinks->type != GGML_TYPE_F32) { + return false; + } + if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) { + return false; + } + if (dst->ne[3] != 1) { + return false; + } + + struct htp_fa_kernel_params kparams; + if (!ggml_hexagon_compute_fa_params(ctx, dst, &kparams)) { + return false; + } + const size_t vtcm_budget = (size_t)ctx->socinfo.vtcm_size_in_mb * 1024 * 1024; + if ((size_t)kparams.vtcm_size > vtcm_budget) { + return false; + } + return true; +} + +// Function pointer table for ggmlhexagon_can_handle_op_through_cdsp. +// Replaces the large switch statement with individual validator functions +// indexed by GGML_OP. Each validator receives the Hexagon context and the +// op tensor; returns true if the DSP can handle the op. +typedef bool (*hexagon_op_validator_t)(ggml_backend_hexagon_context * ctx, const ggml_tensor * op); + +// Binary element-wise ops: ADD, SUB, MUL, DIV +static bool hexagon_validate_binary_op(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + GGML_UNUSED(ctx); + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * src1 = op->src[1]; + const ggml_tensor * dst = op; + if (src0->type == GGML_TYPE_F32) { + if (!src1 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) + return false; + } else if (src0->type == GGML_TYPE_F16) { + if (!src1 || src1->type != GGML_TYPE_F16 || dst->type != GGML_TYPE_F16) + return false; + } else { + return false; + } + if (!ggml_are_same_shape(src0, dst)) return false; + if (!ggml_can_repeat(src1, src0) || ggml_is_permuted(src1)) + return false; + return true; +} + +static bool hexagon_validate_mul_mat(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + return ggmlhexagon_supported_mul_mat(op, ctx); +} + +static bool hexagon_validate_rms_norm(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + GGML_UNUSED(ctx); + const ggml_tensor * src0 = op->src[0]; + if (src0->type != GGML_TYPE_F32 || op->type != GGML_TYPE_F32) + return false; + // Accept non-contiguous views (Qwen3Next per-head reshape) + if (src0->nb[0] != sizeof(float)) + return false; + return true; +} + +// NORM, L2_NORM: dispatched to op_unary (F32, same shape, contiguous dst) +static bool hexagon_validate_norm_op(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + GGML_UNUSED(ctx); + const ggml_tensor * src0 = op->src[0]; + if (src0->type != GGML_TYPE_F32 || op->type != GGML_TYPE_F32) + return false; + if (!ggml_are_same_shape(src0, op)) + return false; + if (!ggml_is_contiguous(op)) + return false; + return true; +} + +// SQR, SQRT: element-wise unary, same as norm_op +static bool hexagon_validate_sqr_sqrt(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + return hexagon_validate_norm_op(ctx, op); +} + +static bool hexagon_validate_rope(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + GGML_UNUSED(ctx); + const int32_t * op_params = &op->op_params[0]; + + // ggml_rope_set_offset: HVX kernels need a VLEN-aligned window start (32 f32 elems) + if (op_params[15] % 32 != 0) { + return false; + } + + int mode = op_params[2]; + if (mode == GGML_ROPE_TYPE_VISION) { + const int n_dims = op_params[1]; + if (n_dims != (int) (op->src[0]->ne[0] / 2)) { + return false; + } + } + if (mode & 1) { + return false; + } + + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * src1 = op->src[1]; + if (src0->type != GGML_TYPE_F32 || op->type != GGML_TYPE_F32) + return false; + if (!src1 || src1->type != GGML_TYPE_I32) + return false; + return true; +} + +static bool hexagon_validate_soft_max(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + GGML_UNUSED(ctx); + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * src1 = op->src[1]; + if (src0->type != GGML_TYPE_F32 || op->type != GGML_TYPE_F32) + return false; + if (src1 != nullptr && src1->type != GGML_TYPE_F16 && src1->type != GGML_TYPE_F32) + return false; + if (op->src[2] != nullptr) + return false; + return true; +} + +static bool hexagon_validate_unary(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + GGML_UNUSED(ctx); + const ggml_tensor * src0 = op->src[0]; + const int unary_op = (int)op->op_params[0]; + switch (unary_op) { + case GGML_UNARY_OP_SILU: + case GGML_UNARY_OP_GELU: + case GGML_UNARY_OP_GELU_QUICK: + case GGML_UNARY_OP_NEG: + case GGML_UNARY_OP_EXP: + case GGML_UNARY_OP_SIGMOID: + case GGML_UNARY_OP_SOFTPLUS: + case GGML_UNARY_OP_TANH: + if (src0->type != GGML_TYPE_F32 || op->type != GGML_TYPE_F32) + return false; + if (ggml_is_permuted(src0)) + return false; + if (!ggml_are_same_shape(src0, op)) + return false; + if (!ggml_is_contiguous(op)) + return false; + return true; + default: + return false; + } +} + +static bool hexagon_validate_glu(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + GGML_UNUSED(ctx); + const ggml_tensor * src0 = op->src[0]; + if (src0->type != GGML_TYPE_F32 || op->type != GGML_TYPE_F32) + return false; + if (!ggml_is_contiguous_1(src0) || !ggml_is_contiguous(op)) + return false; + const int glu_op = (int)op->op_params[0]; + switch (glu_op) { + case GGML_GLU_OP_SWIGLU: + case GGML_GLU_OP_SWIGLU_OAI: + case GGML_GLU_OP_GEGLU: + return true; + default: + return false; + } +} + +static bool hexagon_validate_scale(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + GGML_UNUSED(ctx); + const ggml_tensor * src0 = op->src[0]; + if (src0->type != op->type) return false; + if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) + return false; + return true; +} + +static bool hexagon_validate_cpy(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + GGML_UNUSED(ctx); + const ggml_tensor * src0 = op->src[0]; + if (src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_F32) + return false; + if (op->type != GGML_TYPE_F16 && op->type != GGML_TYPE_F32) + return false; + return true; +} + +static bool hexagon_validate_get_rows(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + GGML_UNUSED(ctx); + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * src1 = op->src[1]; + if (!src1 || src1->type != GGML_TYPE_I32) + return false; + if (src0->type != GGML_TYPE_F32 || op->type != GGML_TYPE_F32) + return false; + return true; +} + +static bool hexagon_validate_set_rows(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + GGML_UNUSED(ctx); + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * src1 = op->src[1]; + if (!src1 || (src1->type != GGML_TYPE_I32 && src1->type != GGML_TYPE_I64)) + return false; + if (src0->type != GGML_TYPE_F32) + return false; + if (op->type != GGML_TYPE_F32 && op->type != GGML_TYPE_F16) + return false; + return true; +} + +static bool hexagon_validate_sum_rows(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + GGML_UNUSED(ctx); + if (op->src[0]->type != GGML_TYPE_F32) + return false; + return true; +} + +static bool hexagon_validate_cont(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + GGML_UNUSED(ctx); + const ggml_tensor * src0 = op->src[0]; + if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) + return false; + if (op->type != GGML_TYPE_F32 && op->type != GGML_TYPE_F16) + return false; + return true; +} + +static bool hexagon_validate_concat(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + GGML_UNUSED(ctx); + const ggml_tensor * src0 = op->src[0]; + if (src0->type != op->type) return false; + if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && + src0->type != GGML_TYPE_I32 && src0->type != GGML_TYPE_I16) + return false; + return true; +} + +static bool hexagon_validate_repeat(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + GGML_UNUSED(ctx); + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * dst = op; + + if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) return false; + + if (src0->type != dst->type) return false; + + if (dst->ne[0] % src0->ne[0] != 0) return false; + if (dst->ne[1] % src0->ne[1] != 0) return false; + if (dst->ne[2] % src0->ne[2] != 0) return false; + if (dst->ne[3] % src0->ne[3] != 0) return false; + + if (ggml_is_transposed(src0) || ggml_is_transposed(dst)) return false; + + return true; +} + +static bool hexagon_validate_diag_mask_inf(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + GGML_UNUSED(ctx); + const ggml_tensor * src0 = op->src[0]; + if (src0->type != GGML_TYPE_F32 || op->type != GGML_TYPE_F32) + return false; + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(op)) + return false; + return true; +} + +static bool hexagon_validate_cumsum(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + GGML_UNUSED(ctx); + if (op->src[0]->type != GGML_TYPE_F32 || op->type != GGML_TYPE_F32) + return false; + return true; +} + +static bool hexagon_validate_diag(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + GGML_UNUSED(ctx); + if (op->src[0]->type != GGML_TYPE_F32 || op->type != GGML_TYPE_F32) + return false; + return true; +} + +static bool hexagon_validate_argsort(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + GGML_UNUSED(ctx); + if (op->src[0]->type != GGML_TYPE_F32) + return false; + return true; +} + +static bool hexagon_validate_pad(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + GGML_UNUSED(ctx); + if (op->src[0]->type != GGML_TYPE_F32) + return false; + return true; +} + +static bool hexagon_validate_im2col(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + GGML_UNUSED(ctx); + const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * dst = op; + const bool is_2D = ((const int32_t *) op->op_params)[6] == 1; + if (!is_2D) { + return false; + } + // F32 image -> F16/F32 columns only + if (src1->type != GGML_TYPE_F32 || (dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_F32)) { + return false; + } + if (!ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) { + return false; + } + // padded im2col stays on CPU; DSP path only covers patch-embed shape + const int32_t p0 = ((const int32_t *) op->op_params)[2]; + const int32_t p1 = ((const int32_t *) op->op_params)[3]; + if (p0 != 0 || p1 != 0) { + return false; + } + return true; +} + +// Qwen3.5-2B delta net: offload conv1d to avoid cgraph split +static bool hexagon_validate_ssm_conv(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + GGML_UNUSED(ctx); + const ggml_tensor * src0 = op->src[0]; + const ggml_tensor * src1 = op->src[1]; + const ggml_tensor * dst = op; + + if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + + if (src0->ne[3] != 1 || src1->ne[2] != 1 || src1->ne[3] != 1 || dst->ne[3] != 1) { + return false; + } + + const int d_conv = src1->ne[0]; + const int d_inner = src0->ne[1]; + const int n_t = dst->ne[1]; + const int n_s = dst->ne[2]; + + if (src0->ne[0] != d_conv - 1 + n_t || src0->ne[1] != d_inner || src0->ne[2] != n_s) { + return false; + } + if (src1->ne[0] != d_conv || src1->ne[1] != d_inner) { + return false; + } + if (dst->ne[0] != d_inner || dst->ne[1] != n_t || dst->ne[2] != n_s) { + return false; + } + if (src0->nb[0] != sizeof(float) || src1->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) { + return false; + } + if (src0->nb[1] != src0->ne[0] * sizeof(float) || src1->nb[1] != src1->ne[0] * sizeof(float)) { + return false; + } + + return true; +} + +// Qwen3.5-2B delta net: offload solve_tri to avoid cgraph split +static bool hexagon_validate_solve_tri(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + GGML_UNUSED(ctx); + const ggml_tensor * src0 = op->src[0]; // A + const ggml_tensor * src1 = op->src[1]; // B + const ggml_tensor * dst = op; // X + if (!src0 || !src1) return false; + if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + if (src0->ne[0] != src0->ne[1]) return false; // A must be square + if (src0->ne[1] != src1->ne[1]) return false; // A.rows == B.rows + if (src0->ne[2] != src1->ne[2] || src0->ne[3] != src1->ne[3]) return false; + if (dst->ne[0] != src1->ne[0] || dst->ne[1] != src1->ne[1] || + dst->ne[2] != src1->ne[2] || dst->ne[3] != src1->ne[3]) { + return false; + } + return true; +} + +static bool hexagon_validate_tri(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + GGML_UNUSED(ctx); + if (op->src[0]->type != GGML_TYPE_F32) + return false; + return true; +} + +static bool hexagon_validate_fill(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + GGML_UNUSED(ctx); + if (op->type != GGML_TYPE_F32 && op->type != GGML_TYPE_F16) + return false; + return true; +} + +static bool hexagon_validate_flash_attn(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + return ggmlhexagon_supported_flash_attn(ctx, op); +} + +static bool hexagon_validate_gated_delta_net(ggml_backend_hexagon_context * ctx, const ggml_tensor * op) { + GGML_UNUSED(ctx); + const struct ggml_tensor * q = op->src[0]; + const struct ggml_tensor * k = op->src[1]; + const struct ggml_tensor * v = op->src[2]; + const struct ggml_tensor * g = op->src[3]; + const struct ggml_tensor * beta = op->src[4]; + const struct ggml_tensor * state = op->src[5]; + const struct ggml_tensor * dst = op; + + if (!q || !k || !v || !g || !beta || !state) { + return false; + } + + if (q->type != GGML_TYPE_F32 || k->type != GGML_TYPE_F32 || v->type != GGML_TYPE_F32 || + g->type != GGML_TYPE_F32 || beta->type != GGML_TYPE_F32 || state->type != GGML_TYPE_F32 || + dst->type != GGML_TYPE_F32) { + return false; + } + + if (!ggml_is_contiguous_rows(q) || !ggml_is_contiguous_rows(k) || !ggml_is_contiguous_rows(v) || + !ggml_is_contiguous(g) || !ggml_is_contiguous(beta) || !ggml_is_contiguous(state) || + !ggml_is_contiguous(dst)) { + return false; + } + + const int64_t S_v = v->ne[0]; + const int64_t H = v->ne[1]; + const int64_t n_tokens = v->ne[2]; + const int64_t n_seqs = v->ne[3]; + const int64_t K = ggml_get_op_params_i32(op, 0); + + if (S_v <= 0 || S_v > 128 || H <= 0 || n_tokens <= 0 || n_seqs <= 0) { + return false; + } + if (q->ne[0] != S_v || k->ne[0] != S_v || q->ne[1] <= 0 || k->ne[1] <= 0 || + q->ne[2] != n_tokens || k->ne[2] != n_tokens || q->ne[3] <= 0 || k->ne[3] <= 0 || + (n_seqs % q->ne[3]) != 0 || (n_seqs % k->ne[3]) != 0) { + return false; + } + if ((g->ne[0] != 1 && g->ne[0] != S_v) || beta->ne[0] != 1) { + return false; + } + if (ggml_nelements(state) != S_v * S_v * H * n_seqs) { + return false; + } + if (dst->ne[0] != S_v * H || dst->ne[1] != n_tokens * n_seqs + S_v * n_seqs * K) { + return false; + } + + return true; +} + +// Static lookup table: one validator per GGML_OP, indexed by op_tensor->op. +// NULL entries mean "not supported on DSP" (returns false). +// Initialized once at first call via init_op_validators(). +static hexagon_op_validator_t s_op_validators[GGML_OP_COUNT]; + +static void init_op_validators(void) { + static bool s_initialized = false; + if (s_initialized) return; + s_initialized = true; + + s_op_validators[GGML_OP_ADD] = hexagon_validate_binary_op; + s_op_validators[GGML_OP_SUB] = hexagon_validate_binary_op; + s_op_validators[GGML_OP_MUL] = hexagon_validate_binary_op; + s_op_validators[GGML_OP_DIV] = hexagon_validate_binary_op; + s_op_validators[GGML_OP_MUL_MAT] = hexagon_validate_mul_mat; + s_op_validators[GGML_OP_RMS_NORM] = hexagon_validate_rms_norm; + s_op_validators[GGML_OP_NORM] = hexagon_validate_norm_op; + s_op_validators[GGML_OP_L2_NORM] = hexagon_validate_norm_op; + s_op_validators[GGML_OP_SQR] = hexagon_validate_sqr_sqrt; + s_op_validators[GGML_OP_SQRT] = hexagon_validate_sqr_sqrt; + s_op_validators[GGML_OP_ROPE] = hexagon_validate_rope; + s_op_validators[GGML_OP_SOFT_MAX] = hexagon_validate_soft_max; + s_op_validators[GGML_OP_UNARY] = hexagon_validate_unary; + s_op_validators[GGML_OP_GLU] = hexagon_validate_glu; + s_op_validators[GGML_OP_SCALE] = hexagon_validate_scale; + s_op_validators[GGML_OP_CPY] = hexagon_validate_cpy; + s_op_validators[GGML_OP_GET_ROWS] = hexagon_validate_get_rows; + s_op_validators[GGML_OP_SET_ROWS] = hexagon_validate_set_rows; + s_op_validators[GGML_OP_SUM_ROWS] = hexagon_validate_sum_rows; + s_op_validators[GGML_OP_SSM_CONV] = hexagon_validate_ssm_conv; + s_op_validators[GGML_OP_CONT] = hexagon_validate_cont; + s_op_validators[GGML_OP_CONCAT] = hexagon_validate_concat; + s_op_validators[GGML_OP_REPEAT] = hexagon_validate_repeat; + s_op_validators[GGML_OP_DIAG_MASK_INF] = hexagon_validate_diag_mask_inf; + s_op_validators[GGML_OP_CUMSUM] = hexagon_validate_cumsum; + s_op_validators[GGML_OP_DIAG] = hexagon_validate_diag; + s_op_validators[GGML_OP_ARGSORT] = hexagon_validate_argsort; + s_op_validators[GGML_OP_PAD] = hexagon_validate_pad; + s_op_validators[GGML_OP_IM2COL] = hexagon_validate_im2col; + s_op_validators[GGML_OP_GATED_DELTA_NET]= hexagon_validate_gated_delta_net; + s_op_validators[GGML_OP_TRI] = hexagon_validate_tri; + s_op_validators[GGML_OP_SOLVE_TRI] = hexagon_validate_solve_tri; + s_op_validators[GGML_OP_FILL] = hexagon_validate_fill; + s_op_validators[GGML_OP_FLASH_ATTN_EXT] = hexagon_validate_flash_attn; +} + +static bool ggmlhexagon_can_handle_op_through_cdsp(ggml_backend_dev_t dev, const struct ggml_tensor * op_tensor) { + if (ggmlhexagon_is_metadata_op(op_tensor->op)) { + return true; + } + + if (!ggmlhexagon_op_buffers_belong_to_dev(dev, op_tensor)) { + return false; + } + + init_op_validators(); + + ggml_backend_hexagon_context * ctx = (ggml_backend_hexagon_context *)dev->context; + hexagon_op_validator_t validator = s_op_validators[op_tensor->op]; + if (validator) { + return validator(ctx, op_tensor); + } + return false; +} + +struct ggml_backend_hexagon_buffer_context { + ~ggml_backend_hexagon_buffer_context() { + if (buffer) { + if (is_ion_buffer) { + if (backend_ctx && backend_ctx->rpc_mempool) { + // Mark the mempool region as free so it can be reused + const char * buf_ptr = (const char *)buffer; + const char * pool_base = (const char *)backend_ctx->rpc_mempool; + if (buf_ptr >= pool_base && buf_ptr < pool_base + (ptrdiff_t)backend_ctx->rpc_mempool_len) { + size_t buf_offset = (size_t)(buf_ptr - pool_base); + for (size_t ri = 0; ri < backend_ctx->ion_regions.size(); ri++) { + auto & r = backend_ctx->ion_regions[ri]; + if (r.in_use && r.offset == buf_offset) { + r.in_use = false; + GGMLHEXAGON_LOG_ALWAYS("[FREE] device=%d region offset=%zu size=%zu", + backend_ctx->device, r.offset, r.size); + // Coalesce with adjacent free regions to reduce external fragmentation + // 1) Merge with next region if free and contiguous + if (ri + 1 < backend_ctx->ion_regions.size()) { + auto & next = backend_ctx->ion_regions[ri + 1]; + if (!next.in_use && r.offset + r.size == next.offset) { + GGMLHEXAGON_LOG_ALWAYS("[FREE] device=%d merge-next: offset=%zu size=%zu + offset=%zu size=%zu -> size=%zu", + backend_ctx->device, r.offset, r.size, next.offset, next.size, r.size + next.size); + r.size += next.size; + backend_ctx->ion_regions.erase( + backend_ctx->ion_regions.begin() + ri + 1); + } + } + // 2) Merge with previous region if free and contiguous + if (ri > 0) { + auto & prev = backend_ctx->ion_regions[ri - 1]; + if (!prev.in_use && prev.offset + prev.size == r.offset) { + GGMLHEXAGON_LOG_ALWAYS("[FREE] device=%d merge-prev: offset=%zu size=%zu + offset=%zu size=%zu -> size=%zu", + backend_ctx->device, prev.offset, prev.size, r.offset, r.size, prev.size + r.size); + prev.size += r.size; + backend_ctx->ion_regions.erase( + backend_ctx->ion_regions.begin() + ri); + } + } + break; + } + } + } + } + } else { + GGMLHEXAGON_LOG_ALWAYS("it shouldn't come here"); + GGML_ASSERT(1 == 0); + } + } + } + + void * buffer = nullptr; + size_t buffer_size = 0; + bool is_ion_buffer= false; + + struct ggml_backend_hexagon_context * backend_ctx = nullptr; +}; + +// Repack quantized types into tiled (HMX) layout only when the tensor +// lives in the repack buffer (e.g. weights loaded from GGUF). Tensors +// in the main buffer (e.g. test-backend-ops allocations) stay in canonical GGML format. +static void ggml_backend_hexagon_buffer_set_tensor(ggml_backend_buffer_t buffer, + ggml_tensor * tensor, const void * data, + size_t offset, size_t size) { + ggml_backend_hexagon_buffer_context * bctx = (ggml_backend_hexagon_buffer_context *)buffer->context; + ggml_backend_hexagon_context * hctx = (ggml_backend_hexagon_context *)buffer->buft->context; + bool is_repack = ggml_backend_buffer_is_hexagon_repack(buffer); + if (is_repack) { + GGMLHEXAGON_LOG_DEBUG("[SET_TENSOR] #%llu name=%s type=%d(%s) ne=[%d,%d,%d,%d] nbytes=%zu is_repack=%d offset=%zu size=%zu\n", + (unsigned long long)hctx->set_tensor_call_count, tensor->name, (int)tensor->type, ggml_type_name(tensor->type), + (int)tensor->ne[0], (int)tensor->ne[1], (int)tensor->ne[2], (int)tensor->ne[3], + ggml_nbytes(tensor), (int)is_repack, offset, size); + } + hctx->set_tensor_call_count++; + + // Track max layer index from the last contiguous digit run in tensor name. + // Supports multiple naming conventions: + // ffn_gate-12 (qwen3, dash separator) + // blk.23.ffn.weight (gemma/llama, dot separator) + // model.layers.5.x (transformers, dot separator) + // One-time cost per tensor during model load, never in graph_compute hot path. + if (tensor->name[0] != '\0') { + const char * name = tensor->name; + size_t name_len = strlen(name); + // find end of last contiguous digit run + const char * digit_end = nullptr; + for (size_t i = name_len; i > 0; --i) { + if (name[i - 1] >= '0' && name[i - 1] <= '9') { + digit_end = name + i; + break; + } + } + if (digit_end) { + // walk back to find start of digit run + const char * digit_start = digit_end; + while (digit_start > name && *(digit_start - 1) >= '0' && *(digit_start - 1) <= '9') { + --digit_start; + } + char * parse_end = nullptr; + long idx = strtol(digit_start, &parse_end, 10); + if (parse_end == digit_end && idx >= 0) { + ggml_backend_hexagon_context * ctx = (ggml_backend_hexagon_context *) buffer->buft->context; + if ((uint32_t)idx > ctx->max_layer_idx_seen) { + ctx->max_layer_idx_seen = (uint32_t)idx; + } + } + } + } + + if (is_repack) { + const char * dp = (const char *)tensor->data; + const char * base = (const char *)hctx->rpc_mempool; + const char * end = base + (ptrdiff_t)hctx->rpc_mempool_len; + GGML_ASSERT(offset == 0); + GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); + + switch (tensor->type) { + case GGML_TYPE_Q4_0: + case GGML_TYPE_IQ4_NL: // identical block layout to Q4_0 + if (dp >= base && dp < end) { + GGMLHEXAGON_LOG_DEBUG("repack"); + repack_q4_0_tiled_to_buf(tensor, data, tensor->data); + } else { + GGMLHEXAGON_LOG_DEBUG("cpu buffer"); + memcpy(tensor->data, data, size); + } + break; + case GGML_TYPE_Q4_1: + if (dp >= base && dp < end) { + GGMLHEXAGON_LOG_DEBUG("repack"); + repack_q4_1_tiled_to_buf(tensor, data, tensor->data); + } else { + GGMLHEXAGON_LOG_DEBUG("cpu buffer"); + memcpy(tensor->data, data, size); + } + break; + case GGML_TYPE_Q8_0: { + if (dp >= base && dp < end) { + GGMLHEXAGON_LOG_DEBUG("repack"); + repack_q8_0_tiled_to_buf(tensor, data, tensor->data); + } else { + GGMLHEXAGON_LOG_DEBUG("cpu buffer"); + memcpy(tensor->data, data, size); + } + break; + } + case GGML_TYPE_MXFP4: + if (dp >= base && dp < end) { + GGMLHEXAGON_LOG_DEBUG("repack"); + repack_mxfp4_tiled_to_buf(tensor, data, tensor->data); + } else { + GGMLHEXAGON_LOG_DEBUG("cpu buffer"); + memcpy(tensor->data, data, size); + } + break; + case GGML_TYPE_BF16: + if (dp >= base && dp < end) { + GGMLHEXAGON_LOG_DEBUG("repack"); + repack_bf16_to_f16(tensor, data); + } else { + GGMLHEXAGON_LOG_DEBUG("cpu buffer"); + memcpy(tensor->data, data, size); + } + break; + case GGML_TYPE_Q4_K: + if (dp >= base && dp < end) { + GGMLHEXAGON_LOG_DEBUG("repack"); + repack_q4k_as_q4_0_tiled_to_buf(tensor, data, tensor->data); + } else { + GGMLHEXAGON_LOG_DEBUG("cpu buffer"); + memcpy(tensor->data, data, size); + } + break; + case GGML_TYPE_Q5_K: { + if (dp >= base && dp < end) { + GGMLHEXAGON_LOG_DEBUG("repack"); + repack_q5k_as_q4_0_tiled_to_buf(tensor, data, tensor->data); + } else { + // weight lives on heap (model too big for mempool); CPU fallback needs raw Q5_K bytes + memcpy(tensor->data, data, size); + } + break; + } + case GGML_TYPE_Q6_K: + if (dp >= base && dp < end) { + GGMLHEXAGON_LOG_DEBUG("repack"); + repack_q6k_as_q4_0_tiled_to_buf(tensor, data, tensor->data); + } else { + GGMLHEXAGON_LOG_DEBUG("cpu buffer"); + memcpy(tensor->data, data, size); + } + break; + default: + GGMLHEXAGON_LOG_DEBUG("cpu buffer"); + memcpy((char *)tensor->data + offset, data, size); + break; + } + } else { + GGMLHEXAGON_LOG_DEBUG("[SET_TENSOR] #%llu name=%s type=%d(%s) ne=[%d,%d,%d,%d] nbytes=%zu is_repack=%d offset=%zu size=%zu\n", + (unsigned long long)hctx->set_tensor_call_count, tensor->name, (int)tensor->type, ggml_type_name(tensor->type), + (int)tensor->ne[0], (int)tensor->ne[1], (int)tensor->ne[2], (int)tensor->ne[3], + ggml_nbytes(tensor), (int)is_repack, offset, size); + memcpy((char *)tensor->data + offset, data, size); + } + + // Mark weights dirty so Phase 9 flushes them on the next batch. + // For repack-buft weights, flush immediately after repack; this moves the + // cold-state cache-clean cost from the first inference batch to model-load + // time. Do NOT clear weights_dirty here: non-repack weights (F32/F16/BF16) + // that were written earlier in the load sequence still need Phase 9 flush. + if (bctx->is_ion_buffer) { + const char * dp = (const char *)tensor->data + offset; + const char * base = (const char *)hctx->rpc_mempool; + const char * end = base + (ptrdiff_t)hctx->rpc_mempool_len; + if (dp >= base && dp < end) { + if (is_repack) { + size_t flush_len = ggml_hexagon_repacked_size(tensor->type, tensor->ne[0], tensor->ne[1], + tensor->ne[2], tensor->ne[3]); + if (flush_len == 0) flush_len = ggml_nbytes(tensor); + cpu_dcache_flush_range(hctx, hctx->rpc_mempool_handle, tensor->data, flush_len); + } else { + hctx->weights_dirty = true; + } + } + } +} + +static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer, + const ggml_tensor * tensor, + void * data, size_t offset, size_t size) { + GGMLHEXAGON_LOG_DEBUG("enter %s", __FUNCTION__); + // Un-repack tiled layout back to canonical GGML format only when + // the tensor lives in the repack buffer. + if (ggml_backend_buffer_is_hexagon_repack(buffer)) { + // In CPU-fallback mode (model exceeds mempool) weights are stored raw, + // so a plain memcpy is the correct read-back. + ggml_backend_hexagon_context * hctx = (ggml_backend_hexagon_context *)buffer->buft->context; + const char * dp = (const char *)tensor->data; + const char * base = (const char *)hctx->rpc_mempool; + const char * end = base + (ptrdiff_t)hctx->rpc_mempool_len; + GGML_ASSERT(offset == 0); + GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); + if (size == ggml_nbytes(tensor)) { + switch (tensor->type) { + case GGML_TYPE_Q4_0: + case GGML_TYPE_IQ4_NL: + if (dp >= base && dp < end) { + GGMLHEXAGON_LOG_DEBUG("unpack"); + repack_tiled_q4_0_to_buf(data, tensor, size); + } else { + GGMLHEXAGON_LOG_DEBUG("cpu buffer"); + memcpy(data, (const char *)tensor->data + offset, size); + } + return; + case GGML_TYPE_Q4_1: + if (dp >= base && dp < end) { + GGMLHEXAGON_LOG_DEBUG("unpack"); + repack_tiled_q4_1_to_buf(data, tensor, size); + } else { + GGMLHEXAGON_LOG_DEBUG("cpu buffer"); + memcpy(data, (const char *)tensor->data + offset, size); + } + return; + case GGML_TYPE_Q8_0: + if (dp >= base && dp < end) { + GGMLHEXAGON_LOG_DEBUG("unpack"); + repack_tiled_q8_0_to_buf(data, tensor, size); + } else { + GGMLHEXAGON_LOG_DEBUG("cpu buffer"); + memcpy(data, (const char *)tensor->data + offset, size); + } + return; + case GGML_TYPE_MXFP4: + if (dp >= base && dp < end) { + GGMLHEXAGON_LOG_DEBUG("unpack"); + repack_tiled_mxfp4_to_buf(data, tensor, size); + } else { + GGMLHEXAGON_LOG_DEBUG("cpu buffer"); + memcpy(data, (const char *)tensor->data + offset, size); + } + return; + case GGML_TYPE_BF16: + if (dp >= base && dp < end) { + GGMLHEXAGON_LOG_DEBUG("unpack"); + repack_f16_to_bf16(tensor, data, size); + } else { + GGMLHEXAGON_LOG_DEBUG("cpu buffer"); + memcpy(data, (const char *)tensor->data + offset, size); + } + return; + case GGML_TYPE_Q4_K: + if (dp >= base && dp < end) { + GGMLHEXAGON_LOG_DEBUG("unpack"); + repack_tiled_q4_0_to_q4k_buf(data, tensor, size); + } else { + GGMLHEXAGON_LOG_DEBUG("cpu buffer"); + memcpy(data, (const char *)tensor->data + offset, size); + } + return; + // Q6_K get_tensor (tiled Q4_0 -> Q6_K) not implemented; + // falls through to raw memcpy. Inference is unaffected since + // the DSP reads the tiled Q4_0 layout directly. + default: + break; + } + } else { + GGMLHEXAGON_LOG_DEBUG("size %zu, nbytes %zu", size, ggml_nbytes(tensor)); + } + } + memcpy(data, (const char *)tensor->data + offset, size); + GGML_UNUSED(buffer); +} + +static void ggml_backend_hexagon_buffer_memset_tensor(ggml_backend_buffer_t buffer, + struct ggml_tensor * tensor, + uint8_t value, size_t offset, size_t size) { + memset((char *)tensor->data + offset, value, size); + + ggml_backend_hexagon_buffer_context * bctx = (ggml_backend_hexagon_buffer_context *)buffer->context; + if (bctx && bctx->is_ion_buffer && bctx->backend_ctx) { + ggml_backend_hexagon_context * hctx = bctx->backend_ctx; + const char * dp = (const char *)tensor->data + offset; + const char * base = (const char *)hctx->rpc_mempool; + if (dp >= base && dp < base + (ptrdiff_t)hctx->rpc_mempool_len) { + hctx->weights_dirty = true; + } + } +} + +static bool ggml_backend_hexagon_buffer_cpy_tensor(ggml_backend_buffer_t buffer, + const struct ggml_tensor * src, + struct ggml_tensor * dst) { + GGML_UNUSED(buffer); + GGML_UNUSED(src); + GGML_UNUSED(dst); + // take the slow path via get/set_tensor (which handles repack/un-repack) + return false; +} + +static void ggml_backend_hexagon_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { + ggml_backend_hexagon_buffer_context * ctx = (ggml_backend_hexagon_buffer_context *)buffer->context; + memset(ctx->buffer, value, ctx->buffer_size); + + if (ctx->is_ion_buffer && ctx->backend_ctx) { + ggml_backend_hexagon_context * hctx = ctx->backend_ctx; + const char * dp = (const char *)ctx->buffer; + const char * base = (const char *)hctx->rpc_mempool; + if (dp >= base && dp < base + (ptrdiff_t)hctx->rpc_mempool_len) { + hctx->weights_dirty = true; + } + } +} + +static void ggml_backend_hexagon_buffer_free_buffer(ggml_backend_buffer_t buffer) { + ggml_backend_hexagon_buffer_context * ctx = (ggml_backend_hexagon_buffer_context *)buffer->context; + // Buffers are freed on model unload. Clear the caches keyed by tensor + // pointers so a reload that reuses the same addresses does not hit stale entries. + struct ggml_backend_hexagon_context * bctx = ctx->backend_ctx; + if (bctx) { + bctx->cgraph_cache.clear(); + bctx->mm_params_cache.clear(); + bctx->ever_dst_ptrs.clear(); + bctx->scratch_cache_ranges.clear(); + bctx->tiled_ion_offsets.clear(); + bctx->warned_non_repack.clear(); + bctx->warned_qkv_name = false; + bctx->set_tensor_call_count = 0; + bctx->dsp_need_weight_inval_reset = true; + } + delete ctx; +} + +static void * ggml_backend_hexagon_buffer_get_base(ggml_backend_buffer_t buffer) { + ggml_backend_hexagon_buffer_context * ctx = (ggml_backend_hexagon_buffer_context *)buffer->context; + return ctx->buffer; +} + +static enum ggml_status ggml_backend_hexagon_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { + ggml_backend_hexagon_buffer_context * ctx = (ggml_backend_hexagon_buffer_context *)buffer->context; + GGML_UNUSED(tensor); + GGML_UNUSED(ctx); + return GGML_STATUS_SUCCESS; +} + +static ggml_backend_buffer_i ggml_backend_hexagon_buffer_interface = { + /* .free_buffer = */ ggml_backend_hexagon_buffer_free_buffer, + /* .get_base = */ ggml_backend_hexagon_buffer_get_base, + /* .init_tensor = */ ggml_backend_hexagon_buffer_init_tensor, + /* .memset_tensor = */ ggml_backend_hexagon_buffer_memset_tensor, + /* .set_tensor = */ ggml_backend_hexagon_buffer_set_tensor, + /* .get_tensor = */ ggml_backend_hexagon_buffer_get_tensor, + /* .set_tensor_2d = */ nullptr, + /* .get_tensor_2d = */ nullptr, + /* .cpy_tensor = */ ggml_backend_hexagon_buffer_cpy_tensor, + /* .clear = */ ggml_backend_hexagon_buffer_clear, + /* .reset = */ nullptr, +}; + +static const char * ggml_backend_hexagon_buffer_type_name(ggml_backend_buffer_type_t buft) { + struct ggml_backend_hexagon_context * ctx = static_cast(buft->context); + if (ctx) { + if (buft == &ctx->repack_buffer_type) { + return ctx->repack_buft_name; + } + return ctx->buft_name; + } + return "hexagon-ion-buffer"; +} + +static ggml_backend_buffer_t ggml_backend_hexagon_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { + struct ggml_backend_hexagon_context * ctx = static_cast(buft->context); + GGML_ASSERT(nullptr != ctx); + GGMLHEXAGON_LOG_ALWAYS("[ALLOC] ENTER device=%d size=%zu bytes (%.2f MiB)", ctx->device, size, (double)size / (1024.0 * 1024.0)); + ggml_backend_hexagon_buffer_context * buffer_ctx = new ggml_backend_hexagon_buffer_context; + buffer_ctx->backend_ctx = ctx; + buffer_ctx->is_ion_buffer = true; + + size_t size_page = 4096; // default page size if sysconf is unavailable +#if defined(__ANDROID__) || defined(__linux__) + long ps = sysconf(_SC_PAGESIZE); + if (ps > 0) size_page = (size_t)ps; +#endif + size_t size_aligned = size; + if (size_page > 0 && 0 != (size_aligned % size_page)) { + size_aligned += (size_page - (size_aligned % size_page)); + } + + GGMLHEXAGON_LOG_ALWAYS("device %d(%s)", ctx->device, ctx->name); + GGML_ASSERT(nullptr != ctx->rpc_mempool); + GGMLHEXAGON_LOG_ALWAYS("device=%d size %ld(%d MiB), rpc_mempool_usage %ld(%d MiB), rpc_mempool_len %ld(%d MiB)", + ctx->device, size, size / SIZE_IN_MB, ctx->rpc_mempool_usage, ctx->rpc_mempool_usage / SIZE_IN_MB, + ctx->rpc_mempool_len, ctx->rpc_mempool_len / SIZE_IN_MB); + + size_t data_limit = ctx->rpc_mempool_len; + + // Try to reuse a free region (best fit) + size_t best_idx = (size_t)-1; + size_t best_waste = (size_t)-1; + for (size_t ri = 0; ri < ctx->ion_regions.size(); ri++) { + const auto & r = ctx->ion_regions[ri]; + if (!r.in_use && r.size >= size_aligned) { + size_t waste = r.size - size_aligned; + if (waste < best_waste) { + best_waste = waste; + best_idx = ri; + } + } + } + + if (best_idx != (size_t)-1) { + // Reuse free region + auto & r = ctx->ion_regions[best_idx]; + buffer_ctx->buffer = (char *)ctx->rpc_mempool + r.offset; + buffer_ctx->buffer_size = size_aligned; // actual requested size, not region size + r.in_use = true; + // split oversized free region to avoid internal fragmentation + // save r before insert; vector insert may invalidate references + size_t r_offset = r.offset; + size_t r_size = r.size; + if (r_size > size_aligned + 128) { + size_t tail_offset = r_offset + size_aligned; + size_t tail_size = r_size - size_aligned; + ctx->ion_regions[best_idx].size = size_aligned; + ion_pool_region tail; + tail.offset = tail_offset; + tail.size = tail_size; + tail.in_use = false; + ctx->ion_regions.insert(ctx->ion_regions.begin() + best_idx + 1, tail); + GGMLHEXAGON_LOG_ALWAYS("[ALLOC] device=%d split region: offset=%zu size=%zu -> used=%zu free=%zu", + ctx->device, r_offset, r_size, size_aligned, tail_size); + } + if (r_offset + size_aligned > ctx->rpc_mempool_usage) { + ctx->rpc_mempool_usage = r_offset + size_aligned; + } + GGMLHEXAGON_LOG_ALWAYS("[ALLOC] device=%d reuse free region: offset=%zu size=%zu (requested=%zu, waste=%zu)", + ctx->device, r_offset, ctx->ion_regions[best_idx].size, size_aligned, + ctx->ion_regions[best_idx].size - size_aligned); + memset(buffer_ctx->buffer, 0, buffer_ctx->buffer_size); + } else { + // Allocate new region from bump allocator tail + size_t aligned_offset = ((ctx->rpc_mempool_usage + 127) / 128) * 128; + if (aligned_offset + size_aligned <= data_limit) { + buffer_ctx->buffer = (char *)ctx->rpc_mempool + aligned_offset; + buffer_ctx->buffer_size = size_aligned; + ctx->rpc_mempool_usage = aligned_offset + size_aligned; + // Record new region + ion_pool_region new_region; + new_region.offset = aligned_offset; + new_region.size = size_aligned; + new_region.in_use = true; + ctx->ion_regions.push_back(new_region); + GGMLHEXAGON_LOG_ALWAYS("[ALLOC] device=%d new region: offset=%zu size=%zu", ctx->device, aligned_offset, size_aligned); + } else { + GGMLHEXAGON_LOG_ALWAYS("device=%d ion pool exhausted: needed %zu MiB, remaining %zu MiB -- falling back to CPU buffer", + ctx->device, size_aligned / SIZE_IN_MB, + (data_limit - ctx->rpc_mempool_usage) / SIZE_IN_MB); + // Do not keep a heap-backed hexagon buft buffer here: the scheduler + // would still assign ops to hexagon, but the per-token mempool mirror + // cannot fit (the pool is full), so compute would fail. Instead return + // a CPU (host) buffer so the scheduler routes these tensors' ops to the + // CPU backend, which reads the raw ggml layout directly. This mirrors + // the documented Q5_K weights -> CPU behavior. + delete buffer_ctx; + return ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size_aligned); + } + } + + if (nullptr == buffer_ctx->buffer) { + GGMLHEXAGON_LOG_ERROR("%s: failed to allocate %d MiB\n", __func__, size / SIZE_IN_MB); + delete buffer_ctx; + return nullptr; + } else { + GGMLHEXAGON_LOG_ALWAYS("%s: succeed to allocate %d MiB\n", __func__, size / SIZE_IN_MB); + } + // Report allocation result and current mempool state + if (buffer_ctx->is_ion_buffer) { + const char * mem_type = "heap"; + const char * data_ptr = (const char *)buffer_ctx->buffer; + const char * ion_base = (const char *)ctx->rpc_mempool; + const char * ion_end = ion_base + ctx->rpc_mempool_len; + if (data_ptr >= ion_base && data_ptr < ion_end) { + mem_type = "mempool"; + } + GGMLHEXAGON_LOG_ALWAYS("[ALLOC] device=%d LEAVE size=%zu (%.2f MiB) -> %s, pool_used=%zu/%zu (%.2f%%)", + ctx->device, size, (double)size / (1024.0 * 1024.0), + mem_type, + ctx->rpc_mempool_usage, ctx->rpc_mempool_len, + ctx->rpc_mempool_len > 0 ? (double)ctx->rpc_mempool_usage * 100.0 / ctx->rpc_mempool_len : 0.0); + } else { + GGMLHEXAGON_LOG_ALWAYS("[ALLOC] device=%d LEAVE size=%zu (%.2f MiB) -> heap", ctx->device, size, (double)size / (1024.0 * 1024.0)); + } + return ggml_backend_buffer_init(buft, ggml_backend_hexagon_buffer_interface, buffer_ctx, size); +} + +static size_t ggml_backend_hexagon_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { + GGML_UNUSED(buft); + //Alignment requirement in bytes + return 128; +} + +static size_t ggml_backend_hexagon_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { + struct ggml_backend_hexagon_context * ctx = static_cast(buft->context); + GGML_ASSERT(nullptr != ctx); + GGML_ASSERT(ctx->rpc_mempool_len > (8 * SIZE_IN_MB)); + return ctx->rpc_mempool_len - (8 * SIZE_IN_MB); +} + +static size_t ggml_backend_hexagon_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor) { + GGML_UNUSED(buft); + // For quantized weight types that will be repacked to tile-based layout + // in set_tensor, allocate enough space for the repacked data. + switch (tensor->type) { + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_MXFP4: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q6_K: + case GGML_TYPE_Q5_K: { + size_t repacked = ggml_hexagon_repacked_size(tensor->type, tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]); + size_t raw = ggml_nbytes(tensor); + return repacked > raw ? repacked : raw; + } + default: + return ggml_nbytes(tensor); + } +} + +static bool ggml_backend_buft_is_hexagon(ggml_backend_buffer_type_t buft) { + return buft->iface.get_name == ggml_backend_hexagon_buffer_type_name; +} + +// Repack buft is identified by comparing the buft pointer against the +// repack_buffer_type member stored in ggml_backend_hexagon_context (both +// main and repack bufts share the same context pointer). Used by +// supports_buft to allow GGML core to route quantized weights through +// set_tensor (which does the in-place tile repack). +static bool ggml_backend_buft_is_hexagon_repack(ggml_backend_buffer_type_t buft) { + auto * ctx = (ggml_backend_hexagon_context *)buft->context; + return buft == &ctx->repack_buffer_type; +} + +static bool ggml_backend_hexagon_buffer_is_host(ggml_backend_buffer_type_t buft) { + GGML_UNUSED(buft); + return true; +} + +// Repack buffer type: is_host=false forces GGML core to call set_tensor +// (which does the repack) instead of reading model data directly into +// tensor->data. Both main and repack buffer types manage the same mempool +// shared memory pool. +static bool ggml_backend_hexagon_repack_buffer_is_host(ggml_backend_buffer_type_t buft) { + GGML_UNUSED(buft); + return false; +} + +// Returns true if this buffer was allocated from the repack buffer type. +// Repack buffers hold quantized weight data in tiled (HMX) layout and +// require set_tensor/get_tensor to repack/unrepack across the boundary. +static bool ggml_backend_buffer_is_hexagon_repack(const struct ggml_backend_buffer * b) { + auto * ctx = (ggml_backend_hexagon_context *)b->buft->context; + return b->buft == &ctx->repack_buffer_type; +} + +// Session consistency check: +// - tensor is null: neutral, accept (compute-temporary-like) +// - tensor has no buffer assigned: neutral, accept (scheduler will route) +// - buffer is hexagon (main or repack) on this device: accept +// - buffer is hexagon on a different device: reject (wrong session) +// - buffer is non-hexagon (e.g. CPU): reject (scheduler should keep on CPU) +// +// Both the main and repack bufts store the owning ggml_backend_hexagon_context +// pointer at buft->context (set in the constructor as `buffer_type.context = this`). +// The current device's context lives at dev->context. +static bool ggmlhexagon_tensor_buffer_is_owned_by(ggml_backend_dev_t dev, const struct ggml_tensor * t) { + if (!t || !t->buffer) { + return true; + } + ggml_backend_buffer_type_t buft = t->buffer->buft; + if (!ggml_backend_buft_is_hexagon(buft) && !ggml_backend_buft_is_hexagon_repack(buft)) { + return false; + } + ggml_backend_hexagon_context * dev_ctx = (ggml_backend_hexagon_context *)dev->context; + ggml_backend_hexagon_context * buft_ctx = (ggml_backend_hexagon_context *)buft->context; + // same logic as ggml_backend_hexagon_device_supports_buft + return buft_ctx->device == dev_ctx->device; +} + +// All srcs and the dst of the op must be mapped to the same hexagon session +// (device). Tensors with no buffer are treated as neutral. Without this, +// the scheduler can incorrectly assign an op to a device whose tensors live +// in another device's mempool region, which would fault on the DSP since +// mempool mappings are not shared across separate FastRPC sessions. +static bool ggmlhexagon_op_buffers_belong_to_dev(ggml_backend_dev_t dev, const struct ggml_tensor * op) { + if (!ggmlhexagon_tensor_buffer_is_owned_by(dev, op)) { + return false; + } + for (int i = 0; i < GGML_MAX_SRC; i++) { + if (!ggmlhexagon_tensor_buffer_is_owned_by(dev, op->src[i])) { + return false; + } + } + return true; +} + +static const char * ggml_backend_hexagon_name(ggml_backend_t backend) { + ggml_backend_hexagon_context * ctx = (ggml_backend_hexagon_context *) backend->context; + return ctx->name; +} + +static void ggml_backend_hexagon_free(ggml_backend_t backend) { + GGMLHEXAGON_LOG_DEBUG("enter %s", __func__ ); + // only delete the backend here; the context (including buffer types) + // must persist across inferences so that model tensors which reference + // buffer types remain valid. The context is owned by the device and + // freed during registry shutdown. + ggml_backend_hexagon_context * ctx = (ggml_backend_hexagon_context *)backend->context; + ctx->backend = nullptr; + delete backend; + + GGMLHEXAGON_LOG_DEBUG("leave %s", __func__ ); +} + +// mempool-based op-batch - packs all ops into mempool shared memory, +// passes only (offset, size) via FastRPC as doorbell. +// avoids FastRPC scatter-gather limits entirely. +static enum ggml_status ggmlhexagon_backend_graph_compute_batch(ggml_backend_t backend, struct ggml_cgraph * cgraph) { + enum ggml_status result = GGML_STATUS_SUCCESS; + int64_t begin_time = ggml_time_us(); + ggml_backend_hexagon_context * ctx = (ggml_backend_hexagon_context *)backend->context; + int64_t gap_from_prev = ctx->last_graph_end_us ? (begin_time - ctx->last_graph_end_us) : 0; + (void)gap_from_prev; // used only in GGMLHEXAGON_LOG_DEBUG + uint32_t graph_n_nodes = (uint32_t)cgraph->n_nodes; + const char * ion_base = (const char *)ctx->rpc_mempool; + const size_t ion_size = ctx->rpc_mempool_len; + size_t saved_mempool_usage = ctx->rpc_mempool_usage; + uint32_t n_tensors = 0; + uint32_t n_ops = 0; + bool cache_hit = false; + + // Snapshot cumulative phase counters at entry so we can compute the + // current call's accounted time at exit (used for unaccounted-time). + int64_t snap_p1 = ctx->cum_p1_us; + int64_t snap_p2 = ctx->cum_p2_us; + int64_t snap_p3 = ctx->cum_p3_us; + int64_t snap_p4 = ctx->cum_p4_us; + int64_t snap_p7 = ctx->cum_p7_us; + int64_t snap_p10 = ctx->cum_p10_us; + + // Phase timing: declare all timers here, used across the pipeline + int64_t t_start, t_p1, t_p2, t_p3, t_p4, t_p5, t_p6, t_p7, t_p8, t_p9, t_p10, t_p11, t_p12; + + // Track temporary mempool regions (mirrors, batch descriptors) allocated in + // this call. Cleanup erases these entries from ion_regions and restores + // rpc_mempool_usage, letting the bump allocator reuse the space next call. + // Note: indices are valid only because graph_compute_batch is single-threaded + // and no other code erases from ion_regions between push_back and final erase. + std::vector temp_region_indices; + + // Storage for cache-miss path; on cache hit we reference cached vectors + // directly to avoid copying ~20-110 KB of descriptors per call. + std::vector local_tensor_src; + std::vector local_hex_ops; + std::vector local_is_weight; + std::vector local_mirror_offset; // per-tensor heap mirror mempool offset, -1 = none + + // supported_nodes is only required to build descriptors on cache miss. + // On a cache hit we restore the derived descriptors directly, skipping this + // second cgraph pass and the vector allocation (the content hash still scans + // every node on every call). + std::vector supported_nodes; + + ggml_backend_hexagon_context::cgraph_cache_entry * cached_entry = nullptr; + + // track per-graph node statistics (what ggml core assigned to this backend) + ctx->total_nodes_processed += graph_n_nodes; + if (ctx->min_nodes_per_graph == 0 || graph_n_nodes < ctx->min_nodes_per_graph) { + ctx->min_nodes_per_graph = graph_n_nodes; + } + if (graph_n_nodes > ctx->max_nodes_per_graph) { + ctx->max_nodes_per_graph = graph_n_nodes; + } + + t_start = ggml_time_us(); + + // ---- Phase 1: collect unique tensor objects + cgraph cache lookup ---- + // Hash over each node's {op, ne[4], nb[4], non-null src[0..GGML_MAX_SRC-1] ptr, + // data ptr, op_params}. NULL srcs contribute nothing; the slot index is folded + // into non-null src values so the same tensor seen at different slots hashes + // differently. op_params is included so two graphs that differ only in node + // parameters (eps/scale/theta etc.) never share cached descriptors. The + // node's own data pointer is kept because distinct dst tensors can share the + // same op/shape/srcs (e.g., buffer reuse), and omitting it caused cache hits + // to restore descriptors pointing to stale tensors. + // cgraph pointer is NOT used: the scheduler rebuilds split->graph every + // call, so the pointer churns. The content is stable. + auto compute_content_hash = [&]() -> uint64_t { + uint64_t h = 0xcbf29ce484222325ULL; // FNV-1a 64-bit offset basis + for (int i = 0; i < cgraph->n_nodes; i++) { + ggml_tensor * node = cgraph->nodes[i]; + if (!node) continue; + h ^= (uint64_t)node->op; h *= 0x100000001b3ULL; + for (int j = 0; j < 4; j++) { h ^= (uint64_t)node->ne[j]; h *= 0x100000001b3ULL; } + for (int j = 0; j < 4; j++) { h ^= (uint64_t)node->nb[j]; h *= 0x100000001b3ULL; } + for (int j = 0; j < GGML_MAX_SRC; j++) { + ggml_tensor * src = node->src[j]; + if (src) { + h ^= (uint64_t)(uintptr_t)src ^ (uint64_t)j; + h *= 0x100000001b3ULL; + } + } + h ^= (uint64_t)(uintptr_t)node->data; h *= 0x100000001b3ULL; + for (int j = 0; j < (int) (GGML_MAX_OP_PARAMS / sizeof(int32_t)); j++) { + h ^= (uint64_t)(uint32_t) node->op_params[j]; h *= 0x100000001b3ULL; + } + } + return h; + }; + + const uint64_t content_hash = compute_content_hash(); + { + auto it = ctx->cgraph_cache.find(content_hash); + if (it != ctx->cgraph_cache.end() && + it->second.n_nodes == cgraph->n_nodes && + it->second.hex_ops.size() > 0) { + cached_entry = &it->second; + cache_hit = true; + ctx->cgraph_cache_hits++; + } else { + ctx->cgraph_cache_misses++; + } + } + + // Bind to cached descriptors on hit, local vectors on miss. This avoids + // the expensive assign() of hex_ops/tensor_src in the hot path. + std::vector & tensor_src = cache_hit ? cached_entry->tensor_src : local_tensor_src; + std::vector & hex_ops = cache_hit ? cached_entry->hex_ops : local_hex_ops; + std::vector & is_weight = cache_hit ? cached_entry->is_weight : local_is_weight; + + if (cache_hit) { + n_tensors = (uint32_t)cached_entry->n_tensors; + n_ops = (uint32_t)cached_entry->n_ops; + } + + if (!cache_hit) { + // ---- collect supported ops (cache miss only) ---- + supported_nodes.reserve(cgraph->n_nodes); + for (int i = 0; i < cgraph->n_nodes; i++) { + ggml_tensor * node = cgraph->nodes[i]; + + if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE + || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW + || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { + continue; + } + + supported_nodes.push_back(node); + } + + if (supported_nodes.empty()) { + return result; + } + + tensor_src.reserve(cgraph->n_nodes); + hex_ops.reserve(supported_nodes.size()); + } + + t_p1 = t_start; t_start = ggml_time_us(); ctx->cum_p1_us += t_start - t_p1; + + // ---- Phase 2: build op descriptors (cache miss only) ---- + if (!cache_hit) { + std::unordered_map tensor_index_map; + tensor_index_map.reserve(cgraph->n_nodes * 2); + auto get_or_add_tensor_idx = [&](ggml_tensor * t) -> int32_t { + if (!t) return -1; + auto it = tensor_index_map.find(t); + if (it != tensor_index_map.end()) return it->second; + int32_t idx = (int32_t)tensor_src.size(); + tensor_index_map[t] = idx; + tensor_src.push_back(t); + return idx; + }; + + for (auto * node : supported_nodes) { + hex_op_desc op; + memset(&op, 0, sizeof(op)); + for (int k = 0; k < 4; k++) op.dst_idx[k] = -1; + for (int k = 0; k < 6; k++) op.src_idx[k] = -1; + op.opcode = node->op; + memcpy(op.params, node->op_params, sizeof(op.params)); + if (node->op == GGML_OP_MUL_MAT) { + ggml_hexagon_precompute_mm_params(ctx, node, op, false); + ctx->n_mul_mat_total_cum++; + if (((const struct htp_mm_kernel_params *) op.kernel_params)->n_hmx) { + ctx->n_hmx_used_cum++; + } + } else if (node->op == GGML_OP_FLASH_ATTN_EXT) { + ggml_hexagon_compute_fa_params(ctx, node, + (struct htp_fa_kernel_params *) op.kernel_params); + } else { + // Unary-family ops (NORM, RMS_NORM, SCALE, SQR, SQRT, UNARY_*, + // L2_NORM, TRI) require host-precomputed htp_unary_kernel_params + // since upstream commit fb30ba9a6. Without this the DSP reads + // zeroed kparams (n_threads=0, etc.) and the output is garbled. + uint32_t unary_htp_op = 0; + if (ggml_op_to_htp_op_unary(node->op, node->op_params, &unary_htp_op)) { + ggml_hexagon_precompute_unary_params(ctx, unary_htp_op, + node->src[0], node->src[1], node, + (struct htp_unary_kernel_params *) op.kernel_params); + op.htp_opcode = (int32_t) unary_htp_op; + } + } + op.src_idx[0] = get_or_add_tensor_idx(node->src[0]); + op.src_idx[1] = (node->src[1]) ? get_or_add_tensor_idx(node->src[1]) : -1; + op.src_idx[2] = (node->src[2]) ? get_or_add_tensor_idx(node->src[2]) : -1; + op.src_idx[3] = (node->src[3]) ? get_or_add_tensor_idx(node->src[3]) : -1; + op.src_idx[4] = (node->src[4]) ? get_or_add_tensor_idx(node->src[4]) : -1; + op.src_idx[5] = (node->src[5]) ? get_or_add_tensor_idx(node->src[5]) : -1; + op.dst_idx[0] = get_or_add_tensor_idx(node); + hex_ops.push_back(op); + } + + n_tensors = (uint32_t)tensor_src.size(); + + GGMLHEXAGON_LOG_DEBUG("mempool-batch %zu ops, %u unique tensors", hex_ops.size(), n_tensors); + if (1 == g_hexagon_appcfg.dump_debug_info) { + for (size_t i = 0; i < hex_ops.size(); i++) { + const hex_op_desc & o = hex_ops[i]; + GGML_UNUSED(o); + GGMLHEXAGON_LOG_DEBUG(" ion-op[%zu] %s: src0[t%d] src1[t%d] src2[t%d] dst[t%d]", + i, ggml_op_name((ggml_op)o.opcode), + o.src_idx[0], o.src_idx[1], o.src_idx[2], o.dst_idx[0]); + } + } + + // Identify weight tensors: src0 of MUL_MAT that is NOT dst of any op. + // Weights are read-only across batches; AP never modifies them per batch, + // so cache flush/invalidate can be skipped for them. + // A tensor that was dst of any op in ANY cgraph (not just this one) is + // not a read-only weight: check the session-global ever-dst set, else + // cross-graph staleness occurs with bit 0 (e.g. qwen3-mtp garble). + { + for (const auto & op : hex_ops) { + uint32_t didx = op.dst_idx[0]; + if (didx < n_tensors) ctx->ever_dst_ptrs.insert(tensor_src[didx]->data); + } + is_weight.assign(n_tensors, 0); + std::vector dst_indices(n_tensors, 0); // indices of tensors that are dst of any op + for (const auto & op : hex_ops) { + uint32_t didx = op.dst_idx[0]; + if (didx < n_tensors) dst_indices[didx] = 1; + } + for (const auto & op : hex_ops) { + if (op.opcode == GGML_OP_MUL_MAT) { + uint32_t sidx = op.src_idx[0]; + if (sidx < n_tensors && !dst_indices[sidx] && + !ctx->ever_dst_ptrs.count(tensor_src[sidx]->data)) { + is_weight[sidx] = 1; + GGMLHEXAGON_LOG_DEBUG("weight-cache: tensor[%d] identified as weight (type=%d)", + sidx, (int)tensor_src[sidx]->type); + } + } + } + } + } // end if (!cache_hit) for Phase 2 + + t_p2 = t_start; t_start = ggml_time_us(); ctx->cum_p2_us += t_start - t_p2; + + // ---- Phase 3: op fusion ---- + // Supported fusions: + // RMS_NORM + MUL -> HTP_OP_RMS_NORM_MUL + // MUL_MAT + ADD -> HTP_OP_MUL_MAT_ADD (bias add inside kernel) + // 3x MUL_MAT (Q,K,V) -> HTP_OP_MUL_MAT_QKV (via DSP execute_op) + // 2x MUL_MAT (gate,up)-> HTP_OP_MUL_MAT_FFN (via DSP execute_op) + // + // QKV/FFN fusion eligibility: + // quantized src0 + F32 src1 + !mm_is_hmx_eligible. + // HMX-eligible MUL_MATs are excluded: fusion redirects to HVX fused + // kernels, while HMX-eligible ops benefit more from the HMX pipeline. + if (!cache_hit) { + // Count src usages of each tensor; fusable intermediates must have exactly one consumer + std::vector src_use_count(n_tensors, 0); + for (const auto & op : hex_ops) { + for (int k = 0; k < 6; k++) { + if (op.src_idx[k] >= 0 && op.src_idx[k] < (int)n_tensors) { + src_use_count[op.src_idx[k]]++; + } + } + } + + std::vector fused_ops; + fused_ops.reserve(hex_ops.size()); + size_t n_rms_norm_mul = 0; + size_t n_mul_mat_add = 0; + size_t n_mul_mat_qkv = 0; + size_t n_mul_mat_ffn = 0; + size_t n_mm_add_skip_use_count = 0; // MUL_MAT+ADD candidate but src_use_count > 1 + size_t n_mm_add_skip_not_adjacent = 0; // MUL_MAT not followed by ADD + size_t n_mm_add_skip_vtcm = 0; // MUL_MAT+ADD candidate but VTCM budget exceeded + + const size_t vtcm_budget = ctx->socinfo.vtcm_size_in_mb * 1024 * 1024; + // QKV/FFN fusion prerequisites: + // - dispatches via DSP execute_op, which provides + // op_matmul_qkv / op_matmul_ffn as dedicated fused kernels (htp/matmul-ops.c). + // htp_arch>=V73 is required because op_matmul_qkv/ffn use HMX instructions. + bool qkv_ffn_enabled = (ctx->socinfo.htp_arch >= V73 && g_hexagon_appcfg.enable_opfusion); + for (size_t i = 0; i < hex_ops.size(); i++) { + hex_op_desc op = hex_ops[i]; + + // RMS_NORM + MUL -> RMS_NORM_MUL + if (op.opcode == GGML_OP_RMS_NORM && i + 1 < hex_ops.size()) { + const hex_op_desc & next = hex_ops[i + 1]; + if (next.opcode == GGML_OP_MUL && + next.src_idx[0] == op.dst_idx[0] && + src_use_count[op.dst_idx[0]] == 1) { + op.htp_opcode = HTP_OP_RMS_NORM_MUL; + op.src_idx[1] = next.src_idx[1]; + op.dst_idx[0] = next.dst_idx[0]; + // Precompute unary kparams for the fused op (src0 = RMS_NORM's + // input, src1 = MUL's other input, dst = MUL's output). + if (op.src_idx[0] >= 0 && op.src_idx[1] >= 0 && op.dst_idx[0] >= 0) { + ggml_hexagon_precompute_unary_params(ctx, HTP_OP_RMS_NORM_MUL, + tensor_src[op.src_idx[0]], tensor_src[op.src_idx[1]], tensor_src[op.dst_idx[0]], + (struct htp_unary_kernel_params *) op.kernel_params); + } + fused_ops.push_back(op); + i++; + n_rms_norm_mul++; + continue; + } + } + + // QKV/FFN fusion: requires DSP execute_op. + if (qkv_ffn_enabled && op.opcode == GGML_OP_MUL_MAT) { + // QKV fusion: 3 MUL_MAT (Q,K,V) -> HTP_OP_MUL_MAT_QKV. + // The Q/K/V MUL_MATs may appear in either Q,K,V or Q,V,K order + // depending on the model (e.g. Gemma4/Llama3 uses Q,K,V, Qwen3 uses Q,V,K). + // Detect the actual order from tensor names and map src/dst accordingly. + // DSP-side expects: src[0]=Wk, src[1]=x, src[2]=Wv, src[3]=Wq; dst[0]=K, dst[1]=V, dst[2]=Q. + if (i + 2 < hex_ops.size()) { + const hex_op_desc & next1 = hex_ops[i + 1]; + const hex_op_desc & next2 = hex_ops[i + 2]; + if (next1.opcode == GGML_OP_MUL_MAT && next2.opcode == GGML_OP_MUL_MAT) { + const ggml_tensor * n_q = tensor_src[op.dst_idx[0]]; + const ggml_tensor * n1 = tensor_src[next1.dst_idx[0]]; + const ggml_tensor * n2 = tensor_src[next2.dst_idx[0]]; + if (is_qkv_mergeable(ctx, n_q, n1, n2)) { + // Determine which of n1/n2 is K and which is V by tensor name. + // Models name their Q/K/V projection outputs as Qcur-* / Kcur-* /Vcur-*. + auto is_k = [](const ggml_tensor * t) { return t && strstr(t->name, "Kcur"); }; + auto is_v = [](const ggml_tensor * t) { return t && strstr(t->name, "Vcur"); }; + + const ggml_tensor * n_k = nullptr; + const ggml_tensor * n_v = nullptr; + (void)n_v; // used in GGMLHEXAGON_LOG_DEBUG + const hex_op_desc * op_k = nullptr; + const hex_op_desc * op_v = nullptr; + bool k_name_match = is_k(n1) && is_v(n2); + bool v_name_match = is_v(n1) && is_k(n2); + bool can_fuse_qkv = true; + if (k_name_match) { + // Q, K, V order (Gemma4, Llama3) + n_k = n1; op_k = &next1; + n_v = n2; op_v = &next2; + } else if (v_name_match) { + // Q, V, K order (Qwen3) + n_k = n2; op_k = &next2; + n_v = n1; op_v = &next1; + } else { + // K and V weights have identical shapes (verified by + // is_qkv_mergeable), so structural check cannot tell K + // from V. Skip fusion to avoid silent misordering. + if (!ctx->warned_qkv_name) { + ctx->warned_qkv_name = true; + GGMLHEXAGON_LOG_ALWAYS( + "QKV fusion: cannot identify K/V by tensor name " + "(n1='%s', n2='%s'), skipping fusion. " + "Expected names containing 'Kcur'/'Vcur'.", + n1->name[0] ? n1->name : "?", + n2->name[0] ? n2->name : "?"); + } + can_fuse_qkv = false; + } + + if (can_fuse_qkv) { + struct htp_mm_kernel_params kparams; + ggml_hexagon_precompute_fused_qkv_params(ctx, n_k->src[0], n_k->src[1], &kparams); + if ((size_t)kparams.vtcm_size <= vtcm_budget) { + int32_t wq_idx = op.src_idx[0]; + int32_t x_idx = op.src_idx[1]; + int32_t q_dst = op.dst_idx[0]; + op.htp_opcode = HTP_OP_MUL_MAT_QKV; + op.src_idx[0] = op_k->src_idx[0]; // Wk + op.src_idx[1] = x_idx; // x (shared) + op.src_idx[2] = op_v->src_idx[0]; // Wv + op.src_idx[3] = wq_idx; // Wq + op.dst_idx[0] = op_k->dst_idx[0]; // K + op.dst_idx[1] = op_v->dst_idx[0]; // V + op.dst_idx[2] = q_dst; // Q + op.dst_idx[3] = -1; + memcpy(op.kernel_params, &kparams, sizeof(kparams)); + fused_ops.push_back(op); + i += 2; + n_mul_mat_qkv++; + ctx->n_fused_qkv_cum++; + GGMLHEXAGON_LOG_DEBUG("DBG QKV fusion: q=%s k=%s v=%s | Wq[t%d] Wk[t%d] Wv[t%d] x[t%d] | Q[t%d] K[t%d] V[t%d]", + n_q->name ? n_q->name : "?", + n_k->name ? n_k->name : "?", + n_v->name ? n_v->name : "?", + wq_idx, op_k->src_idx[0], op_v->src_idx[0], x_idx, + q_dst, op_k->dst_idx[0], op_v->dst_idx[0]); + continue; + } else { + GGMLHEXAGON_LOG_INFO("skip QKV fusion: VTCM needed (%d) > budget (%zu)", (int)kparams.vtcm_size, vtcm_budget); + } + } + } + } + } + + // FFN fusion: 2 MUL_MAT (gate,up) -> HTP_OP_MUL_MAT_FFN. + // Current op is gate, next is up. + // src0=Wgate, src1=y, src2=Wup; dst[0]=gate, dst[1]=up. + // Only triggers when is_mergeable_mul_mat returns true + // (quantized src0 + F32 src1 + !mm_is_hmx_eligible). + if (i + 1 < hex_ops.size()) { + const hex_op_desc & next = hex_ops[i + 1]; + if (next.opcode == GGML_OP_MUL_MAT) { + const ggml_tensor * n_gate = tensor_src[op.dst_idx[0]]; + const ggml_tensor * n_up = tensor_src[next.dst_idx[0]]; + if (is_mergeable_mul_mat_pair(ctx, n_gate, n_up)) { + struct htp_mm_kernel_params kparams; + ggml_hexagon_precompute_fused_ffn_params(ctx, n_gate->src[0], n_gate->src[1], &kparams); + if ((size_t)kparams.vtcm_size <= vtcm_budget) { + op.htp_opcode = HTP_OP_MUL_MAT_FFN; + // src0=Wgate (keep), src1=y (keep) + op.src_idx[2] = next.src_idx[0]; // Wup + op.src_idx[3] = -1; + // dst[0]=gate (keep) + op.dst_idx[1] = next.dst_idx[0]; // up + op.dst_idx[2] = -1; + op.dst_idx[3] = -1; + memcpy(op.kernel_params, &kparams, sizeof(kparams)); + fused_ops.push_back(op); + i += 1; + n_mul_mat_ffn++; + ctx->n_fused_ffn_cum++; + GGMLHEXAGON_LOG_DEBUG("DBG FFN fusion: gate=%s up=%s | Wgate[t%d] y[t%d] Wup[t%d] | gate[t%d] up[t%d]", + n_gate->name ? n_gate->name : "?", + n_up->name ? n_up->name : "?", + op.src_idx[0], op.src_idx[1], next.src_idx[0], + op.dst_idx[0], next.dst_idx[0]); + continue; + } else { + GGMLHEXAGON_LOG_DEBUG("skip FFN fusion: VTCM needed (%d) > budget (%zu)", + (int)kparams.vtcm_size, vtcm_budget); + } + } + } + } + } + + // MUL_MAT + ADD -> MUL_MAT_ADD (bias add inside matmul kernel) + // Only applies to pre-norm models where MUL_MAT (down_proj) + // is immediately followed by residual ADD. Gemma uses post-norm + // (MUL_MAT -> RMS_NORM -> MUL -> ADD), so this won't trigger there. + // + // Bias (src2) is read from DDR, not VTCM, so the kparams from + // Phase 2 (MUL_MAT) are reusable. The VTCM budget check is + // defensive: if the matmul already saturates VTCM as a plain + // MUL_MAT, fusing into MUL_MAT_ADD cannot save anything; better + // to keep it as 2 separate ops than risk silent overflow. + if (op.opcode == GGML_OP_MUL_MAT && i + 1 < hex_ops.size()) { + const hex_op_desc & next = hex_ops[i + 1]; + if (next.opcode == GGML_OP_ADD && + (next.src_idx[0] == op.dst_idx[0] || next.src_idx[1] == op.dst_idx[0])) { + if (src_use_count[op.dst_idx[0]] != 1) { + n_mm_add_skip_use_count++; + } else { + int32_t bias_idx = -1; + if (next.src_idx[0] == op.dst_idx[0]) { + bias_idx = next.src_idx[1]; + } else if (next.src_idx[1] == op.dst_idx[0]) { + bias_idx = next.src_idx[0]; + } + if (bias_idx >= 0) { + const struct htp_mm_kernel_params * kparams_mm = + (const struct htp_mm_kernel_params *) op.kernel_params; + if ((size_t) kparams_mm->vtcm_size <= vtcm_budget) { + op.htp_opcode = HTP_OP_MUL_MAT_ADD; + op.src_idx[2] = bias_idx; + op.dst_idx[0] = next.dst_idx[0]; + fused_ops.push_back(op); + i++; + n_mul_mat_add++; + ctx->n_fused_mm_add_cum++; + continue; + } else { + GGMLHEXAGON_LOG_INFO("skip MUL_MAT_ADD fusion: VTCM needed (%d) > budget (%zu)", + (int) kparams_mm->vtcm_size, vtcm_budget); + n_mm_add_skip_vtcm++; (void)n_mm_add_skip_vtcm; + } + } + } + } else { + n_mm_add_skip_not_adjacent++; + } + } + + fused_ops.push_back(op); + } + + if (n_rms_norm_mul + n_mul_mat_add + n_mul_mat_qkv + n_mul_mat_ffn > 0) { + GGMLHEXAGON_LOG_DEBUG("op-fusion: %zu ops -> %zu ops (%zu RMS_NORM_MUL, %zu MUL_MAT_ADD, %zu MUL_MAT_QKV, %zu MUL_MAT_FFN)", + hex_ops.size(), fused_ops.size(), + n_rms_norm_mul, n_mul_mat_add, n_mul_mat_qkv, n_mul_mat_ffn); + hex_ops = std::move(fused_ops); + } + if (n_mm_add_skip_use_count > 0 || n_mm_add_skip_not_adjacent > 0) { + GGMLHEXAGON_LOG_DEBUG("mm_add fusion diag: skip_use_count=%zu skip_not_adjacent=%zu", + n_mm_add_skip_use_count, n_mm_add_skip_not_adjacent); + } + } // end if (!cache_hit) for Phase 3 + + n_ops = (uint32_t)hex_ops.size(); + + // ---- Cache save: store Phase 1/2/3 result keyed by content_hash ---- + // Only on miss. operator[] safely creates entry if absent; on hit we + // already restored from cache, so skip the assign work entirely. + if (!cache_hit) { + auto & entry = ctx->cgraph_cache[content_hash]; + entry.content_hash = content_hash; + entry.n_nodes = cgraph->n_nodes; + entry.n_tensors = (int)n_tensors; + entry.n_ops = (int)n_ops; + entry.tensor_src.assign(tensor_src.begin(), tensor_src.end()); + entry.supported_nodes.assign(supported_nodes.begin(), supported_nodes.end()); + entry.hex_ops.assign(hex_ops.begin(), hex_ops.end()); + entry.is_weight.assign(is_weight.begin(), is_weight.end()); + // Bound the cache to avoid unbounded growth across many distinct graphs. + // unordered_map iteration order is arbitrary, so evict the true + // FIFO-oldest entry by sequence number instead of erase(begin()). + // Linear scan is fine: eviction is rare and the map holds at most + // CGRAPH_CACHE_MAX + 1 entries. + entry.insert_seq = ++ctx->cgraph_cache_seq; + if (ctx->cgraph_cache.size() > ctx->CGRAPH_CACHE_MAX) { + auto oldest = ctx->cgraph_cache.begin(); + for (auto it = ctx->cgraph_cache.begin(); it != ctx->cgraph_cache.end(); ++it) { + if (it->second.insert_seq < oldest->second.insert_seq) { + oldest = it; + } + } + ctx->cgraph_cache.erase(oldest); + } + } + + t_p3 = t_start; t_start = ggml_time_us(); ctx->cum_p3_us += t_start - t_p3; + + // ---- Phase 4: compute layout sizes ---- + const uint32_t hdr_size = (uint32_t)sizeof(hex_batch_hdr); // 24 bytes + const uint32_t ops_region = (uint32_t)(n_ops * sizeof(hex_op_desc)); // 240*N + const uint32_t tens_region = (uint32_t)(n_tensors * sizeof(hex_tensor_desc)); // 112*M + // align ops/tensors regions to HEX_OP_ALIGN (128B) for DSP cache-line/DMA friendliness + const uint32_t ops_offset = (hdr_size + HEX_OP_ALIGN - 1) & ~(HEX_OP_ALIGN - 1); + const uint32_t tensors_offset = ops_offset + ((ops_region + HEX_OP_ALIGN - 1) & ~(HEX_OP_ALIGN - 1)); + const uint32_t total_desc_size = tensors_offset + tens_region; + + t_p4 = t_start; t_start = ggml_time_us(); ctx->cum_p4_us += t_start - t_p4; + + // AP-side guard: fail early if batch exceeds DSP static array limits. + // Values must stay in sync with htp/entry.c. The tensor limit fails the + // batch; the weight limit only warns because entry.c degrades gracefully + // when WEIGHT_INVAL_MAX_PTRS overflows (per-use invalidate). + enum { + AP_DSP_MAX_TENSORS = 4096, // DSP_OPT_MAX_TENSORS in entry.c + AP_DSP_MAX_WEIGHTS = 4096, // WEIGHT_INVAL_MAX_PTRS in entry.c + }; + if (n_tensors > AP_DSP_MAX_TENSORS) { + GGMLHEXAGON_LOG_ERROR("mempool-batch: n_tensors=%u exceeds DSP limit %u; reduce graph size or split batch", + n_tensors, (uint32_t)AP_DSP_MAX_TENSORS); + return GGML_STATUS_FAILED; + } + if (n_ops > (uint32_t)AP_DSP_MAX_TENSORS * 4) { + // entry.c rejects n_ops > DSP_OPT_MAX_TENSORS * 4 + GGMLHEXAGON_LOG_ERROR("mempool-batch: n_ops=%u exceeds DSP limit %d", n_ops, AP_DSP_MAX_TENSORS * 4); + return GGML_STATUS_FAILED; + } + { + uint32_t n_weights = 0; + for (uint32_t i = 0; i < n_tensors; i++) { + n_weights += is_weight[i] ? 1u : 0u; + } + if (n_weights > (uint32_t)AP_DSP_MAX_WEIGHTS) { + GGMLHEXAGON_LOG_WARN("mempool-batch: %u weights exceed WEIGHT_INVAL_MAX_PTRS %d; " + "DSP will re-invalidate weights on every use (perf degraded)", + n_weights, AP_DSP_MAX_WEIGHTS); + } + } + + // ---- Phase 5: handle heap tensors -> mirror into mempool ---- + int64_t t_prev = ggml_time_us(); + // Three-step approach: + // Step 1: Collect unique data pointers and compute max mirror size per buffer + // Step 2: Allocate one mirror per unique buffer (not per tensor) + // Step 3: Build per-tensor mirror offset lookup and mirrors list for copy-back + // This ensures: (a) shared buffers get one mirror with max size, + // (b) each tensor descriptor gets correct ne/nb. + // + // Mirrors are keyed by unique data pointer only; no separate mirror is + // allocated for in-place ops. Tensors sharing a pointer (e.g. in-place + // op src0 and dst) share one mirror: DSP reads and writes the same + // region, so in-place semantics hold. Phase 9 flush, Phase 11 invalidate + // and Phase 12 copy-back all cover the full mirror, keeping the heap + // copy coherent. + struct ion_mirror { + int32_t tensor_idx; + void * original_data; + uint32_t mirror_offset; + uint32_t data_len; + }; + std::vector mirrors; + + // Step 1: Collect unique data pointers and max sizes + struct buffer_mirror_info { + uint32_t mirror_offset; + uint32_t max_data_len; + bool allocated; + }; + std::unordered_map buffer_mirrors_map; + + for (int32_t tidx = 0; tidx < (int32_t)n_tensors; tidx++) { + ggml_tensor * t = tensor_src[tidx]; + if (!t->data) continue; + + const char * data_ptr = (const char *)t->data; + if (data_ptr >= ion_base && data_ptr < ion_base + (ptrdiff_t)ion_size) { + continue; // already in mempool + } + + uint32_t t_size = (uint32_t)ggml_nbytes(t); + // Defensive: set_tensor only repacks weights living inside the pool, so a + // heap-resident quantized weight keeps raw layout in practice (the scheduler + // routes those ops to the CPU backend). Should one arrive pre-repacked, + // mirror its full repacked extent; Phase 6 flags it via warned_non_repack. + bool is_quant_weight = is_weight[tidx] && t->type != GGML_TYPE_F32 && t->type != GGML_TYPE_F16 && t->type != GGML_TYPE_BF16; + if (is_quant_weight) { + size_t repacked = ggml_hexagon_repacked_size(t->type, t->ne[0], t->ne[1], t->ne[2], t->ne[3]); + if (repacked > 0) t_size = (uint32_t)repacked; + } + // Tensors sharing a data pointer are views of the same parent ggml + // buffer, so reading/writing the largest view's extent never crosses + // the parent allocation. This invariant underpins the shared-mirror + // sizing here and the max-len copy-back in Phase 12. + auto it = buffer_mirrors_map.find(t->data); + if (it == buffer_mirrors_map.end()) { + buffer_mirrors_map[t->data] = {0, t_size, false}; + } else if (t_size > it->second.max_data_len) { + it->second.max_data_len = t_size; + } + } + + // Step 2: Allocate mirrors for each unique data pointer + size_t data_limit = ion_size; + for (auto & kv : buffer_mirrors_map) { + void * data_ptr = kv.first; + buffer_mirror_info & info = kv.second; + size_t mirror_size = info.max_data_len; + size_t aligned_offset = (ctx->rpc_mempool_usage + 127u) & ~127u; + + if (aligned_offset + mirror_size > data_limit) { + GGMLHEXAGON_LOG_ERROR("mempool-batch: mempool full for mirror (%zu bytes)", mirror_size); + std::sort(temp_region_indices.begin(), temp_region_indices.end(), std::greater()); + for (size_t ri : temp_region_indices) { + ctx->ion_regions.erase(ctx->ion_regions.begin() + ri); + } + ctx->rpc_mempool_usage = saved_mempool_usage; + return GGML_STATUS_ALLOC_FAILED; + } + + uint32_t moff = (uint32_t)aligned_offset; + void * ion_buf = (char *)ctx->rpc_mempool + moff; + ctx->rpc_mempool_usage = aligned_offset + mirror_size; + + // Record mirror as a temporary mempool region + ion_pool_region mirror_region; + mirror_region.offset = aligned_offset; + mirror_region.size = mirror_size; + mirror_region.in_use = true; + ctx->ion_regions.push_back(mirror_region); + temp_region_indices.push_back(ctx->ion_regions.size() - 1); + + memcpy(ion_buf, data_ptr, mirror_size); + + info.mirror_offset = moff; + info.allocated = true; + + GGMLHEXAGON_LOG_DEBUG("mempool-batch: mirror buffer %p -> mempool offset=0x%x (%u bytes)", + data_ptr, moff, info.max_data_len); + } + + // Step 3: Build per-tensor mirror offset lookup and mirrors list for copy-back. + local_mirror_offset.assign(n_tensors, -1); + for (int32_t tidx = 0; tidx < (int32_t)n_tensors; tidx++) { + ggml_tensor * t = tensor_src[tidx]; + if (!t->data) continue; + + const char * data_ptr = (const char *)t->data; + if (data_ptr >= ion_base && data_ptr < ion_base + (ptrdiff_t)ion_size) { + continue; // already in mempool + } + + auto it = buffer_mirrors_map.find(t->data); + if (it == buffer_mirrors_map.end() || !it->second.allocated) continue; + + local_mirror_offset[tidx] = (int32_t)it->second.mirror_offset; + + ion_mirror m; + m.tensor_idx = tidx; + m.original_data = t->data; + m.mirror_offset = it->second.mirror_offset; + m.data_len = it->second.max_data_len; // use full allocated (possibly repacked) size for flush/copy-back + mirrors.push_back(m); + } + + t_p5 = ggml_time_us() - t_prev; t_prev = ggml_time_us(); + + // ---- Phase 6: track mempool offsets for repacked quantized weights ---- + // weights are already repacked to tile-based layout + // by set_tensor during model loading. Phase 6 only tracks mempool + // offsets for DSP descriptor updates in Phase 8. + { + // Quantized weights (Q4_0 / Q4_1 / Q8_0 / IQ4_NL / MXFP4 / Q4_K / Q5_K / Q6_K) + // are repacked to tile-based (HMX) layout in set_tensor during model loading. + // By the time graph_compute_batch runs, every quantized weight's + // data at t->data is already in tiled layout, so Phase 6 does + // NO repack work here. + // + // The only thing Phase 6 still needs to do is record the mempool offset + // of each pool-resident repacked weight in tiled_ion_offsets so Phase 8 + // can build the DSP descriptor with the correct data_offset. Only stable + // (in-mempool) offsets are cached; see the recording site below. Any + // quantized weight that somehow lives outside the repack buft is logged + // as a one-shot warning (should not happen with the current model loader). + for (uint32_t i = 0; i < n_tensors; i++) { + ggml_tensor * t = tensor_src[i]; + if (!t || !t->data) continue; + bool is_quant_weight = is_weight[i] && t->type != GGML_TYPE_F32 && t->type != GGML_TYPE_F16 && t->type != GGML_TYPE_BF16; + if (!is_quant_weight) continue; + if (t->type != GGML_TYPE_Q4_0 && t->type != GGML_TYPE_Q4_1 && + t->type != GGML_TYPE_Q8_0 && t->type != GGML_TYPE_IQ4_NL && + t->type != GGML_TYPE_Q4_K && t->type != GGML_TYPE_Q5_K && t->type != GGML_TYPE_Q6_K && + t->type != GGML_TYPE_MXFP4) continue; + const int32_t K = t->ne[0]; + if (K % 32 != 0 || K <= 0) continue; + + if (!t->buffer || !ggml_backend_buffer_is_hexagon_repack(t->buffer)) { + if (ctx->warned_non_repack.insert(t->data).second) { + GGMLHEXAGON_LOG_WARN("tiled: weight %s (data=%p) not in repack buft; " + "assuming set_tensor already repacked it", + t->name, t->data); + } + } + + if (ctx->tiled_ion_offsets.find(t->data) != ctx->tiled_ion_offsets.end()) { + continue; // already recorded on a prior graph_compute call + } + + // Record the offset for Phase 8 - pool-resident weights only. + // Heap-mirrored quantized weights are deliberately skipped: their + // mirror offset changes per call, so a cached value would point at + // unrelated pool data. Phase 8 then leaves such tensors on the + // generic descriptor path with the fresh mirror offset. + const char * dp = (const char *)t->data; + if (dp >= ion_base && dp < ion_base + (ptrdiff_t)ion_size) { + ctx->tiled_ion_offsets[t->data] = (uint32_t)(dp - ion_base); + } + } + } + + t_p6 = ggml_time_us() - t_prev; t_prev = ggml_time_us(); + + // ---- Phase 7: allocate batch descriptor region in mempool ---- + size_t batch_align = HEX_BATCH_ALIGN; + size_t batch_offset_raw = ctx->rpc_mempool_usage; + size_t batch_offset_aligned = (batch_offset_raw + batch_align - 1) & ~(batch_align - 1); + + if (batch_offset_aligned + total_desc_size > data_limit) { + GGMLHEXAGON_LOG_ERROR("mempool-batch: mempool full for batch desc (%zu bytes at offset %zu)", + total_desc_size, batch_offset_aligned); + // free temp mirrors (descending order to keep indices valid), rollback bump + std::sort(temp_region_indices.begin(), temp_region_indices.end(), std::greater()); + for (size_t ri : temp_region_indices) { + ctx->ion_regions.erase(ctx->ion_regions.begin() + ri); + } + ctx->rpc_mempool_usage = saved_mempool_usage; + return GGML_STATUS_ALLOC_FAILED; + } + + uint32_t batch_offset = (uint32_t)batch_offset_aligned; + ctx->rpc_mempool_usage = batch_offset_aligned + total_desc_size; + // Record batch descriptor as a temporary mempool region + ion_pool_region batch_region; + batch_region.offset = batch_offset_aligned; + batch_region.size = total_desc_size; + batch_region.in_use = true; + ctx->ion_regions.push_back(batch_region); + temp_region_indices.push_back(ctx->ion_regions.size() - 1); + + t_p7 = t_prev; t_prev = ggml_time_us(); ctx->cum_p7_us += t_prev - t_p7; + + // ---- Phase 8: build descriptors directly in mempool ---- + t_prev = ggml_time_us(); + uint8_t *ion_batch = (uint8_t *)ctx->rpc_mempool + batch_offset; + hex_batch_hdr * hdr = (hex_batch_hdr *)ion_batch; + memset(hdr, 0, sizeof(*hdr)); + hdr->n_ops = n_ops; + hdr->n_tensors = n_tensors; + hdr->ops_offset = ops_offset; + hdr->tensors_offset = tensors_offset; + hdr->total_size = total_desc_size; + + // write op descriptors + hex_op_desc * ops_out = (hex_op_desc *)(ion_batch + ops_offset); + memcpy(ops_out, hex_ops.data(), ops_region); + + // write tensor descriptors with computed offsets + hex_tensor_desc * tens_out = (hex_tensor_desc *)(ion_batch + tensors_offset); + for (uint32_t i = 0; i < n_tensors; i++) { + ggml_tensor * t = tensor_src[i]; + hex_tensor_desc * td = &tens_out[i]; + + // weights may be stored in a different format (see set_tensor); + // the DSP only knows the storage-type kernels + td->type = (int32_t)ggml_hexagon_weight_dsp_type(t->type); + td->ne[0] = (int32_t)t->ne[0]; td->ne[1] = (int32_t)t->ne[1]; + td->ne[2] = (int32_t)t->ne[2]; td->ne[3] = (int32_t)t->ne[3]; + td->nb[0] = (int32_t)t->nb[0]; td->nb[1] = (int32_t)t->nb[1]; + td->nb[2] = (int32_t)t->nb[2]; td->nb[3] = (int32_t)t->nb[3]; + memcpy(td->op_params, t->op_params, sizeof(td->op_params)); + td->data_len = (uint32_t)ggml_nbytes(t); + + const char * data_ptr = (const char *)t->data; + if (data_ptr >= ion_base && data_ptr < ion_base + (ptrdiff_t)ion_size) { + // mempool tensor: direct offset + td->data_offset = (uint32_t)(data_ptr - ion_base); + td->flags = is_weight[i] ? 2 : 0; // 2=weight (skip cache flush) + } else { + // heap tensor: look up mempool offset + int32_t moff = local_mirror_offset[i]; + if (moff >= 0) { + td->data_offset = (uint32_t)moff; + td->flags = 1; // writable (mirrored) + } else { + // No mirror means t->data was NULL (Phase 5 mirrors every + // non-pool tensor). Offset 0 would alias the first pool region, + // so refuse the batch instead of letting the DSP stomp weights. + GGMLHEXAGON_LOG_ERROR("mempool-batch: tensor[%d] has no data and no mirror; aborting batch", i); + std::sort(temp_region_indices.begin(), temp_region_indices.end(), std::greater()); + for (size_t ri : temp_region_indices) { + ctx->ion_regions.erase(ctx->ion_regions.begin() + ri); + } + ctx->rpc_mempool_usage = saved_mempool_usage; + return GGML_STATUS_FAILED; + } + } + + // tiled: update descriptor to match tile-based repacked layout + // (repack done in set_tensor during model loading via repack buffer type) + bool is_quant_weight = is_weight[i] && t->type != GGML_TYPE_F32 && t->type != GGML_TYPE_F16 && t->type != GGML_TYPE_BF16; + if (is_quant_weight) { + auto it = ctx->tiled_ion_offsets.find(t->data); + if (it != ctx->tiled_ion_offsets.end()) { + const int32_t ne0_p = (int32_t)hex_round_up((uint32_t)t->ne[0], 32); + const int32_t ne1_p = (int32_t)hex_round_up((uint32_t)t->ne[1], 32); + td->ne[0] = ne0_p; + td->ne[1] = ne1_p; + // nb[1] is used by DSP DMA as weight_stride = weight + nc * nb[1]. + // In the tiled layout, tiles are stored as (ne1_p/32) x (ne0_p/32) + // tiles of tile_size bytes each. The byte offset to column nc is + // (nc/32) * (ne0_p/32) * tile_size, so nb[1] = (ne0_p/32) * tile_size / 32. + // ggml_row_size gives the original (non-tiled) stride, which is wrong here. + td->nb[1] = (int32_t)((ne0_p / 32) * htp_mm_get_weight_tile_size((int)ggml_hexagon_weight_dsp_type(t->type)) / 32); + td->nb[2] = td->nb[1] * ne1_p; + td->nb[3] = td->nb[2] * (int32_t)t->ne[2]; + td->data_len = (uint32_t)ggml_hexagon_repacked_size(t->type, t->ne[0], t->ne[1], t->ne[2], t->ne[3]); + td->data_offset = it->second; + td->flags = 2; // weight (skip cache flush after first-touch) + } + } + } + + GGMLHEXAGON_LOG_DEBUG("mempool-batch: submitted offset=0x%x size=%u (%u ops, %u tensors)", batch_offset, total_desc_size, n_ops, n_tensors); + + // ion_sync_mode controls which cache coherency mechanism to use: + // 0 = both DC CVAC/CIVAC + DMA_BUF_IOCTL_SYNC (safest) + // 1 = ion_sync only (skip manual DC CVAC/CIVAC, rely on kernel DMA_BUF_IOCTL_SYNC; default) + // 2 = DC CVAC/CIVAC only (skip ion_sync, manual cache maintenance only) + const bool do_dc_cvac = (g_hexagon_appcfg.ion_sync_mode != 1); + const bool do_ion_sync = (g_hexagon_appcfg.ion_sync_mode != 2); + + t_p8 = ggml_time_us() - t_prev; t_prev = ggml_time_us(); + + // ---- Phase 9: AP -> DSP cache coherency ---- + + // Flush CPU cache to DRAM so DSP can read AP-written data. + { + // ion_sync_mode=1 path: skip the per-tensor/cgraph range scans + // entirely; the DMA_BUF_IOCTL_SYNC below handles cache coherency + // for the whole mempool. + if (!do_dc_cvac) { + if (do_ion_sync) { + int ion_fd = ctx->rpc_mempool_handle; + if (ion_fd > 0) ion_sync_for_direction(ion_fd, 1); + } + int was_weights_dirty = ctx->weights_dirty ? 1 : 0; + ctx->weights_dirty = false; + GGMLHEXAGON_LOG_DEBUG("mempool-batch: phase9 skipped (ion_sync_mode=%d) dirty=%d", + g_hexagon_appcfg.ion_sync_mode, was_weights_dirty); + (void)was_weights_dirty; + } else { + // Collect per-tensor dirty ranges and flush them individually (merged). + // A single continuous [min, max] range would also flush the holes + // between low-offset activations and high-offset weights (~940MB of + // wasted cache-line writes when only a few MB are actually dirty). + std::vector> ranges; + ranges.reserve(n_tensors + mirrors.size() + (size_t)cgraph->n_nodes + 4); + + // Track data pointers already flushed to avoid duplicate ranges + // from the cgraph->nodes loop (which overlaps heavily with tensor_src). + std::unordered_set flushed_ptrs; + flushed_ptrs.reserve(n_tensors + 16); + + // Diagnostic counters (per-call, reset every batch) so we can see which + // source is dominating the flush cost. Logged once per call. + uint64_t dbg_bytes_tensor = 0, dbg_bytes_mirror = 0; + uint64_t dbg_bytes_repack_ion = 0, dbg_bytes_batch = 0, dbg_bytes_cgraph = 0; + uint32_t dbg_ranges_tensor = 0, dbg_ranges_cgraph = 0; + (void)dbg_bytes_tensor; (void)dbg_bytes_mirror; + (void)dbg_bytes_repack_ion; (void)dbg_bytes_batch; (void)dbg_bytes_cgraph; + (void)dbg_ranges_tensor; (void)dbg_ranges_cgraph; + + auto add_range = [&](uint32_t off, uint32_t len) { + if (len > 0) ranges.push_back({off, off + len}); + }; + + for (uint32_t i = 0; i < n_tensors; i++) { + ggml_tensor * t = tensor_src[i]; + if (!t || !t->data) continue; + if (is_weight[i] && !ctx->weights_dirty) continue; + const char * dp = (const char *)t->data; + if (dp >= ion_base && dp < ion_base + (ptrdiff_t)ion_size) { + // For quantized weights repacked in-place by set_tensor, + // use the repacked size (larger than ggml_nbytes). + bool is_quant_weight = is_weight[i] && t->type != GGML_TYPE_F32 && t->type != GGML_TYPE_F16 && t->type != GGML_TYPE_BF16; + size_t flush_size = is_quant_weight ? ggml_hexagon_repacked_size(t->type, t->ne[0], t->ne[1], t->ne[2], t->ne[3]) : ggml_nbytes(t); + if (flush_size == 0) flush_size = ggml_nbytes(t); + add_range((uint32_t)(dp - ion_base), (uint32_t)flush_size); + flushed_ptrs.insert(t->data); + dbg_bytes_tensor += flush_size; + dbg_ranges_tensor++; + } + } + for (const auto & m : mirrors) { + add_range(m.mirror_offset, m.data_len); + dbg_bytes_mirror += m.data_len; + } + add_range(batch_offset, total_desc_size); + dbg_bytes_batch += total_desc_size; + + // Also flush non-op tensors in cgraph not in tensor_src (e.g., test sentinels). + // Without this, Phase 11 DC CIVAC can invalidate cache lines containing + // unflushed sentinel data, causing sentinel mismatch. + // + // Skip repack-buft weights when weights_dirty is false. + // The per-tensor loop above already guards against re-flushing clean + // weights via is_weight; this loop did not, which caused every + // graph_compute call to re-flush the entire repack weight region + // even though no repack had happened. + for (int i = 0; i < cgraph->n_nodes; i++) { + ggml_tensor * t = cgraph->nodes[i]; + if (!t || !t->data) continue; + if (flushed_ptrs.count(t->data)) continue; // already covered by tensor_src loop + bool is_repack_buf = t->buffer && ggml_backend_buffer_is_hexagon_repack(t->buffer); + if (!ctx->weights_dirty && is_repack_buf) { + continue; // repack-buft weight, cache already coherent + } + const char * dp = (const char *)t->data; + if (dp >= ion_base && dp < ion_base + (ptrdiff_t)ion_size) { + bool is_qw = is_repack_buf && + t->type != GGML_TYPE_F32 && t->type != GGML_TYPE_F16 && t->type != GGML_TYPE_BF16; + size_t sz = is_qw ? ggml_hexagon_repacked_size(t->type, t->ne[0], t->ne[1], t->ne[2], t->ne[3]) : ggml_nbytes(t); + if (sz == 0) sz = ggml_nbytes(t); + add_range((uint32_t)(dp - ion_base), (uint32_t)sz); + dbg_bytes_cgraph += sz; + dbg_ranges_cgraph++; + } + } + + uint32_t flush_bytes = 0; + uint32_t n_flush = 0; + (void)flush_bytes; (void)n_flush; + if (do_dc_cvac && !ranges.empty()) { + std::sort(ranges.begin(), ranges.end()); + // Merge overlapping/adjacent ranges. Merge gap = 1 cache line (64B): + // flushing a tiny gap is cheaper than issuing a second flush call. + const uint32_t merge_gap = 64; + uint32_t cur_start = ranges[0].first; + uint32_t cur_end = ranges[0].second; + for (size_t i = 1; i < ranges.size(); i++) { + if (ranges[i].first <= cur_end || ranges[i].first - cur_end <= merge_gap) { + if (ranges[i].second > cur_end) cur_end = ranges[i].second; + } else { + cpu_dcache_flush_range(ctx, 0, (char *)ctx->rpc_mempool + cur_start, cur_end - cur_start); + flush_bytes += cur_end - cur_start; + n_flush++; + cur_start = ranges[i].first; + cur_end = ranges[i].second; + } + } + cpu_dcache_flush_range(ctx, 0, (char *)ctx->rpc_mempool + cur_start, cur_end - cur_start); + flush_bytes += cur_end - cur_start; + n_flush++; + } + (void)flush_bytes; (void)n_flush; + (void)dbg_bytes_tensor; (void)dbg_bytes_mirror; + (void)dbg_bytes_repack_ion; (void)dbg_bytes_batch; (void)dbg_bytes_cgraph; + (void)dbg_ranges_tensor; (void)dbg_ranges_cgraph; + // Also try DMA_BUF_IOCTL_SYNC as extra safeguard + if (do_ion_sync) { + int ion_fd = ctx->rpc_mempool_handle; + if (ion_fd > 0) ion_sync_for_direction(ion_fd, 1); + } + + int was_weights_dirty = ctx->weights_dirty ? 1 : 0; + ctx->weights_dirty = false; + (void)was_weights_dirty; + } // end else (do_dc_cvac) + } + + t_p9 = ggml_time_us() - t_prev; t_prev = ggml_time_us(); + + // ---- Phase 10: FastRPC doorbell call (only 2 scalars!) ---- + // 3-way split for fine-grained perf: + // rpc_setup: AP-side work between Phase 9 end and invoke() entry + // dsp_exec: the synchronous invoke() call itself (RPC round-trip + // + DSP-side work + DSP->AP reply) + // civac: AP-side cache invalidate after invoke() returns + // (measured in Phase 11 below, accumulated into cum_p10_civac_us) + ctx->rpc_batch_call_count++; + + // If a model was unloaded since last batch, reset DSP first-touch weight + // tracking so new weights at reused ION addresses get properly invalidated. + if (ctx->dsp_need_weight_inval_reset) { + const uint32_t mode_bits = (uint32_t)g_hexagon_appcfg.dsp_cache_mode & 0xFu; + const uint32_t trace_bit0 = (g_hexagon_appcfg.dsp_cache_trace_bit0 ? 0x10000u : 0u); + const uint32_t trace_bit1 = (g_hexagon_appcfg.dsp_cache_trace_bit1 ? 0x20000u : 0u); + const uint32_t reset_bit = 0x10u; + const uint32_t reset_payload = mode_bits | trace_bit0 | trace_bit1 | reset_bit; + int rst_err = ggml_dsp_execute_batch(ctx->ggmlop_handle, reset_payload, 0xFFFC); + if (AEE_SUCCESS != rst_err) { + GGMLHEXAGON_LOG_ERROR("DSP weight_inval reset failed: 0x%x, aborting batch", rst_err); + // Free temp regions (mirrors, batch desc) and rollback mempool bump + std::sort(temp_region_indices.begin(), temp_region_indices.end(), std::greater()); + for (size_t ri : temp_region_indices) { + ctx->ion_regions.erase(ctx->ion_regions.begin() + ri); + } + ctx->rpc_mempool_usage = saved_mempool_usage; + return GGML_STATUS_FAILED; + } else { + ctx->dsp_need_weight_inval_reset = false; + GGMLHEXAGON_LOG_DEBUG("DSP weight_inval array reset (model reload)"); + } + } + + int64_t t_p10_pre = ggml_time_us(); + int hexagon_error = ggml_dsp_execute_batch(ctx->ggmlop_handle, batch_offset, total_desc_size); + int64_t t_p10_post = ggml_time_us(); + + if (AEE_SUCCESS != hexagon_error) { + GGMLHEXAGON_LOG_WARN("ggml_dsp_execute_batch failed: 0x%x", hexagon_error); + result = GGML_STATUS_FAILED; + } + + // t_p10 captures the entire synchronous invoke (== old p10 minus civac) + t_p10 = t_p10_post - t_p10_pre; + ctx->cum_p10_us += t_p10; + ctx->cum_p10_dsp_exec_us += t_p10; + // rpc_setup = AP-side cost between Phase 9 end and the invoke entry + int64_t p10_rpc_setup = t_p10_pre - t_prev; + ctx->cum_p10_rpc_setup_us += p10_rpc_setup; + t_prev = ggml_time_us(); + + // ---- Phase 11: invalidate CPU cache for DSP-written mempool regions ---- + // civac time is tracked separately via t_civac (accumulated into + // cum_p10_civac_us), keeping the invalidate cost visible apart from + // p10 (sync invoke) and p12 (copy-back). + int64_t t_civac = ggml_time_us(); // civac start + // DSP writes results to DRAM via mempool buffer, but CPU cache may still hold + // stale data. DC CIVAC + ion_sync controlled by ion_sync_mode (see Phase 9). + if (hexagon_error == AEE_SUCCESS) { + if (!do_dc_cvac) { + // ion_sync_mode=1: rely solely on DMA_BUF_IOCTL_SYNC. + if (do_ion_sync) { + int ion_fd = ctx->rpc_mempool_handle; + if (ion_fd > 0) ion_sync_for_direction(ion_fd, 0); + } + } else { + // collect per-dst invalidate ranges (same as Phase 9 flush); + // do not use a single min..max sweep - it invalidates holes between outputs + // iterate all dst_idx slots (fused ops like QKV write K,V,Q to dst_idx[0..2]) + // reuse scratch vector to avoid per-token heap alloc + auto & inval_ranges = ctx->scratch_cache_ranges; + inval_ranges.clear(); + inval_ranges.reserve(n_ops * 2); + for (uint32_t oi = 0; oi < n_ops; oi++) { + const hex_op_desc & cur_op = hex_ops[oi]; + for (int di = 0; di < HTP_OP_MAX_OUTPUTS; di++) { + int32_t dst_idx = cur_op.dst_idx[di]; + if (dst_idx < 0) continue; + if ((uint32_t)dst_idx >= n_tensors) continue; + ggml_tensor * dst_t = tensor_src[dst_idx]; + if (!dst_t || !dst_t->data) continue; + + uint32_t dst_off = 0xFFFFFFFFu; + const char * dp = (const char *)dst_t->data; + if (dp >= ion_base && dp < ion_base + (ptrdiff_t)ion_size) { + dst_off = (uint32_t)(dp - ion_base); + } else { + int32_t moff = local_mirror_offset[dst_idx]; + if (moff >= 0) dst_off = (uint32_t)moff; + } + if (dst_off == 0xFFFFFFFFu) continue; + + size_t dst_len = ggml_nbytes(dst_t); + uint32_t start = dst_off & ~63u; + // 64-bit arithmetic to avoid overflow when dst_off + dst_len + 63 > UINT32_MAX + uint32_t end = (uint32_t)(((uint64_t)dst_off + (uint64_t)dst_len + 63u) & ~63ull); + inval_ranges.push_back({start, end}); + } + } + if (!inval_ranges.empty()) { + std::sort(inval_ranges.begin(), inval_ranges.end()); + const uint32_t merge_gap = 64; + uint32_t cur_start = inval_ranges[0].first; + uint32_t cur_end = inval_ranges[0].second; + uint32_t total_inval = 0; + for (size_t i = 1; i < inval_ranges.size(); i++) { + if (inval_ranges[i].first <= cur_end || inval_ranges[i].first - cur_end <= merge_gap) { + if (inval_ranges[i].second > cur_end) cur_end = inval_ranges[i].second; + } else { + cpu_dcache_inval_range(ctx, 0, + (const char *)ctx->rpc_mempool + cur_start, cur_end - cur_start); + total_inval += cur_end - cur_start; + cur_start = inval_ranges[i].first; + cur_end = inval_ranges[i].second; + } + } + cpu_dcache_inval_range(ctx, 0, + (const char *)ctx->rpc_mempool + cur_start, cur_end - cur_start); + total_inval += cur_end - cur_start; + GGMLHEXAGON_LOG_DEBUG("mempool-batch: phase11 DC CIVAC %zu ranges, %u bytes total", + inval_ranges.size(), total_inval); + (void)total_inval; + } + // Also try DMA_BUF_IOCTL_SYNC as extra safeguard + if (do_ion_sync) { + int ion_fd = ctx->rpc_mempool_handle; + if (ion_fd > 0) ion_sync_for_direction(ion_fd, 0); + } + } // end else (do_dc_cvac) + } + + // record civac time (Phase 11 only). Accumulated into cum_p10_civac_us. + { + int64_t civac_us = ggml_time_us() - t_civac; + ctx->cum_p10_civac_us += civac_us; + } + + // Reset bump pointer so next graph_compute reuses the same mempool region. + // Without this, rpc_mempool_usage only grows and eventually exhausts the pool, + // causing mirror alloc failure (data_offset=0 -> DSP corrupts model weights). + ctx->rpc_mempool_usage = saved_mempool_usage; + + // Phase 11 timing capture (includes mempool bump reset above) + t_p11 = ggml_time_us() - t_prev; t_prev = ggml_time_us(); + ctx->cum_p11_us += t_p11; + + // ---- Phase 12: copy-back mirrored results to heap ---- + if (hexagon_error == AEE_SUCCESS && !mirrors.empty()) { + // One copy-back per unique pointer, sized to the largest sharing view. + // Safe under the same-parent-buffer invariant noted in Phase 5 Step 1: + // aliases beyond the DSP-written range hold bytes identical to the heap + // copy, so rewriting them is a no-op. + std::unordered_map> copyback_map; + for (const auto & m : mirrors) { + auto it = copyback_map.find(m.original_data); + if (it == copyback_map.end()) { + copyback_map[m.original_data] = {m.mirror_offset, m.data_len}; + } else { + if (m.data_len > it->second.second) { + it->second.second = m.data_len; + } + } + } + for (const auto & kv : copyback_map) { + void * orig_data = kv.first; + uint32_t moff = kv.second.first; + uint32_t max_len = kv.second.second; + memmove(orig_data, (const char *)ctx->rpc_mempool + moff, max_len); + } + } + + // free temp regions (mirrors, batch desc); erase in descending order so + // indices stay valid. Dead entries must be removed to prevent unbounded + // vector growth and best-fit scan slowdown. + std::sort(temp_region_indices.begin(), temp_region_indices.end(), std::greater()); + for (size_t ri : temp_region_indices) { + ctx->ion_regions.erase(ctx->ion_regions.begin() + ri); + } + + t_p12 = ggml_time_us() - t_prev; + + int64_t end_time = ggml_time_us(); + int64_t graph_dur = end_time - begin_time; + // (cum_p10_us is already updated at the end of the Phase 10 invoke() + // block; do not add t_p10 a second time here.) + + // compute unaccounted time (wall-clock not covered by phase timers above) + // Use entry snapshots so the subtraction yields this call's contribution only. + { + int64_t accounted_this_call = (ctx->cum_p1_us - snap_p1) + + (ctx->cum_p2_us - snap_p2) + + (ctx->cum_p3_us - snap_p3) + + (ctx->cum_p4_us - snap_p4) + + (ctx->cum_p7_us - snap_p7) + + (ctx->cum_p10_us - snap_p10) + + t_p5 + t_p6 + t_p8 + t_p9 + t_p11 + t_p12; + int64_t unaccounted = graph_dur - accounted_this_call; + if (unaccounted < 0) unaccounted = 0; // guard against measurement noise + ctx->cum_unaccounted_us += unaccounted; + } + + // update cumulative stats + ctx->cumulative_graph_us += graph_dur; + ctx->last_graph_end_us = end_time; + + // per-phase cumulative time: p1-p4, p7, p10 (incl. its 3-way split) and + // p11 accumulate inline at their phase; p5, p6, p8, p9, p12 use the + // trailing accumulators below. + ctx->cum_p5_us += t_p5; + ctx->cum_p6_us += t_p6; + ctx->cum_p8_us += t_p8; + ctx->cum_p9_us += t_p9; + ctx->cum_p12_us += t_p12; + + // per-call min/max + if (ctx->min_p10_us == 0 || t_p10 < ctx->min_p10_us) ctx->min_p10_us = t_p10; + if (t_p10 > ctx->max_p10_us) ctx->max_p10_us = t_p10; + if (ctx->min_n_ops_per_call == 0 || n_ops < ctx->min_n_ops_per_call) { + ctx->min_n_ops_per_call = n_ops; + } + if (n_ops > ctx->max_n_ops_per_call) { + ctx->max_n_ops_per_call = n_ops; + } + + { + int64_t rpc_overhead = graph_dur - t_p10; + if (rpc_overhead < 0) rpc_overhead = 0; + if (ctx->min_rpc_overhead_us == 0 || rpc_overhead < ctx->min_rpc_overhead_us) { + ctx->min_rpc_overhead_us = rpc_overhead; + } + if (rpc_overhead > ctx->max_rpc_overhead_us) { + ctx->max_rpc_overhead_us = rpc_overhead; + } + ctx->sum_rpc_overhead_us += rpc_overhead; + } + + if (ctx->min_graph_us == 0 || graph_dur < ctx->min_graph_us) ctx->min_graph_us = graph_dur; + + if (graph_dur > ctx->max_graph_us) { + ctx->max_graph_us = graph_dur; + ctx->max_graph_n_nodes = graph_n_nodes; + ctx->max_graph_n_ops = n_ops; + GGMLHEXAGON_LOG_DEBUG("new max graph_dur=%lld us (n_nodes=%u n_ops=%u p10=%lld p9=%lld p11=%lld)", + (long long)graph_dur, graph_n_nodes, n_ops, + (long long)t_p10, (long long)t_p9, (long long)t_p11); + } + GGMLHEXAGON_LOG_DEBUG("mempool-batch timing: p5=%lld p6=%lld p8=%lld p9=%lld p10=%lld p11=%lld p12=%lld (us) ops=%u", + (long long)t_p5, (long long)t_p6, (long long)t_p8, (long long)t_p9, + (long long)t_p10, (long long)t_p11, (long long)t_p12, n_ops); + GGMLHEXAGON_LOG_DEBUG("graph n_ops %u", n_ops); + GGMLHEXAGON_LOG_DEBUG("graph inference duration %lld microseconds (gap_from_prev=%lld us)", (long long)graph_dur, (long long)gap_from_prev); + GGMLHEXAGON_LOG_DEBUG("rpc stats: batch_calls=%llu cum_p10=%lld us cum_graph=%lld us avg_p10=%lld us avg_graph=%lld us", + (unsigned long long)ctx->rpc_batch_call_count, + (long long)ctx->cum_p10_us, (long long)ctx->cumulative_graph_us, + ctx->rpc_batch_call_count ? (long long)(ctx->cum_p10_us / (int64_t)ctx->rpc_batch_call_count) : 0, + ctx->rpc_batch_call_count ? (long long)(ctx->cumulative_graph_us / (int64_t)ctx->rpc_batch_call_count) : 0); + + return result; +} + +// Reorder cgraph nodes to improve DSP VTCM cache locality. +// Stack MUL_MAT ops sharing the same src1 (input activation) so the DSP can +// reuse VTCM-resident dynamically quantized src1 across consecutive matmuls. +// +// Fusion pairs recognized by Phase 3 inline fusion in graph_compute_batch +// (RMS_NORM+MUL, MUL_MAT+ADD) are kept adjacent so the inline fusion still +// triggers. Only independent MUL_MAT groups (single node, quantized src0) are +// eligible for reordering. +static void ggml_backend_hexagon_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * gf) { + GGML_ASSERT(backend); + GGML_ASSERT(gf); + + const int n = gf->n_nodes; + if (n < 2) { + return; + } + + // Step 1: mark fusion pairs (Phase 3 patterns). Nodes sharing group_id + // must stay adjacent and in order so Phase 3 can still detect (i, i+1). + std::vector group_id(n, -1); + int next_group = 0; + int n_mm_add_groups = 0; // count of MUL_MAT+ADD fusion pairs found + int n_rms_mul_groups = 0; // count of RMS_NORM+MUL fusion pairs found + (void)n_mm_add_groups; (void)n_rms_mul_groups; // used in GGMLHEXAGON_LOG_DEBUG + for (int i = 0; i < n; i++) { + if (group_id[i] != -1) { + continue; + } + struct ggml_tensor * node = gf->nodes[i]; + + if (node->op == GGML_OP_RMS_NORM && i + 1 < n) { + struct ggml_tensor * next = gf->nodes[i + 1]; + if (next->op == GGML_OP_MUL && next->src[0] == node) { + group_id[i] = next_group; + group_id[i + 1] = next_group; + next_group++; + n_rms_mul_groups++; + i++; + continue; + } + } + + if (node->op == GGML_OP_MUL_MAT && i + 1 < n) { + struct ggml_tensor * next = gf->nodes[i + 1]; + if (next->op == GGML_OP_ADD && + (next->src[0] == node || next->src[1] == node)) { + group_id[i] = next_group; + group_id[i + 1] = next_group; + next_group++; + n_mm_add_groups++; + i++; + continue; + } + } + } + + GGMLHEXAGON_LOG_DEBUG("graph_optimize: n_nodes=%d n_mm_add_groups=%d n_rms_mul_groups=%d", + n, n_mm_add_groups, n_rms_mul_groups); + + // Step 2: build group list (each group is 1 or 2 contiguous node indices). + std::vector> groups; + { + std::vector visited(n, false); + for (int i = 0; i < n; i++) { + if (visited[i]) { + continue; + } + std::vector g; + if (group_id[i] != -1) { + for (int j = i; j < n; j++) { + if (group_id[j] == group_id[i]) { + g.push_back(j); + visited[j] = true; + } + } + } else { + g.push_back(i); + visited[i] = true; + } + groups.push_back(std::move(g)); + } + } + + // Step 3: reorder. Move stackable MUL_MAT groups with the same src1 close + // together via a forward 16-group window. Non-stackable groups stay put. + auto is_stackable_mul_mat = [](const struct ggml_tensor * node) -> bool { + if (node == nullptr) { + return false; + } + if (node->op != GGML_OP_MUL_MAT && node->op != GGML_OP_MUL_MAT_ID) { + return false; + } + return node->src[0] && ggml_is_quantized(node->src[0]->type); + }; + + auto same_src1 = [](const struct ggml_tensor * a, const struct ggml_tensor * b) -> bool { + return a->src[1] != nullptr && a->src[1] == b->src[1]; + }; + + std::vector new_node_order; + new_node_order.reserve(n); + std::vector group_used(groups.size(), false); + constexpr int N_FORWARD = 16; + + for (size_t g0 = 0; g0 < groups.size(); g0++) { + if (group_used[g0]) { + continue; + } + group_used[g0] = true; + for (int idx : groups[g0]) { + new_node_order.push_back(idx); + } + + if (groups[g0].size() != 1) { + continue; + } + const struct ggml_tensor * node0 = gf->nodes[groups[g0][0]]; + if (!is_stackable_mul_mat(node0)) { + continue; + } + + for (size_t g1 = g0 + 1; g1 < groups.size() && g1 <= g0 + N_FORWARD; g1++) { + if (group_used[g1] || groups[g1].size() != 1) { + continue; + } + const struct ggml_tensor * node1 = gf->nodes[groups[g1][0]]; + if (!is_stackable_mul_mat(node1) || !same_src1(node0, node1)) { + continue; + } + group_used[g1] = true; + for (int idx : groups[g1]) { + new_node_order.push_back(idx); + } + } + } + + // Step 4: write back reordered nodes. Only order changes; tensor pointers + // remain valid, so all src/dst links stay intact. + std::vector new_nodes(n); + for (int i = 0; i < n; i++) { + new_nodes[i] = gf->nodes[new_node_order[i]]; + } + for (int i = 0; i < n; i++) { + gf->nodes[i] = new_nodes[i]; + } +} + +static const char * ggml_backend_hexagon_device_get_description(ggml_backend_dev_t dev) { + GGML_UNUSED(dev); + return "HTP"; +} + +static const char * ggml_backend_hexagon_device_get_name(ggml_backend_dev_t dev) { + struct ggml_backend_hexagon_context * ctx = static_cast(dev->context); + if (nullptr == ctx) { + GGMLHEXAGON_LOG_ALWAYS("pls check why ctx is null"); + return "unknown"; + } + return ctx->name; +} + +static void ggml_backend_hexagon_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { + struct ggml_backend_hexagon_context * ctx = static_cast(dev->context); + if ((nullptr == ctx) || (ctx->device >= GGML_HEXAGON_MAX_DEVICES)) { + GGMLHEXAGON_LOG_ALWAYS("pls check params"); + *free = 0; + *total = 0; + return; + } + + GGMLHEXAGON_LOG_WARN("get_memory: enter device=%d domain_id=%d", ctx->device, ctx->domain_id); + + // ggml backend has domain_id == -1 (not a real CDSP PD) + if (-1 == ctx->domain_id) { + *total = ggmlhexagon_get_system_total_memory_in_bytes(); + *free = ggmlhexagon_get_system_free_memory_in_bytes(); + } else { + size_t rpc_ion_memsize = 0; + size_t rpc_ion_usage = 0; + rpc_ion_memsize = ctx->rpc_mempool_capacity; + rpc_ion_usage = ctx->rpc_mempool_usage; + *total = rpc_ion_memsize; + *free = (rpc_ion_memsize - rpc_ion_usage); + GGMLHEXAGON_LOG_WARN("get_memory: device %d, rpc memsize %d MiB, usage %d MiB, free %d MiB", + ctx->device, rpc_ion_memsize / SIZE_IN_MB, + rpc_ion_usage / SIZE_IN_MB, (rpc_ion_memsize - rpc_ion_usage) / SIZE_IN_MB); + } +} + +static enum ggml_backend_dev_type ggml_backend_hexagon_device_get_type(ggml_backend_dev_t dev) { + GGML_UNUSED(dev); + return GGML_BACKEND_DEVICE_TYPE_GPU; +} + +static void ggml_backend_hexagon_device_get_props(ggml_backend_dev_t dev, + struct ggml_backend_dev_props * props) { + props->name = ggml_backend_hexagon_device_get_name(dev); + props->description = ggml_backend_hexagon_device_get_description(dev); + props->type = ggml_backend_hexagon_device_get_type(dev); + props->device_id = nullptr; // no PCI bus id for Hexagon CDSP devices + ggml_backend_hexagon_device_get_memory(dev, &props->memory_free, &props->memory_total); + props->caps = { + /* .async = */ false, + /* .host_buffer = */ false, + /* .buffer_from_host_ptr = */ false, + /* .events = */ false, + /* .mmap_support = */ false, + }; +} + +static ggml_backend_i ggml_backend_hexagon_interface = { + /* .get_name = */ ggml_backend_hexagon_name, + /* .free = */ ggml_backend_hexagon_free, + /* .set_tensor_async = */ nullptr, + /* .get_tensor_async = */ nullptr, + /* .set_tensor_2d_async = */ nullptr, + /* .get_tensor_2d_async = */ nullptr, + /* .cpy_tensor_async = */ nullptr, + /* .synchronize = */ nullptr, + /* .graph_plan_create = */ nullptr, + /* .graph_plan_free = */ nullptr, + /* .graph_plan_update = */ nullptr, + /* .graph_plan_compute = */ nullptr, + /* .graph_compute = */ ggmlhexagon_backend_graph_compute_batch, + /* .event_record = */ nullptr, + /* .event_wait = */ nullptr, + /* .graph_optimize = */ nullptr, +}; + +static ggml_guid_t ggml_backend_hexagon_guid() { + static ggml_guid guid = { 0x7b, 0x57, 0xdc, 0xaf, 0xde, 0x12, 0x1d, 0x49, + 0x12, 0x12, 0x12, 0x12, 0x12, 0x12, 0x12, 0x12 }; + return &guid; +} + +struct ggml_backend_hexagon_reg_context { + std::vector devices; + ~ggml_backend_hexagon_reg_context() { + for (auto * dev : devices) { + // context persists across inferences (not freed in ggml_backend_hexagon_free), + // so it is always present and must be cleaned up here. + if (dev->context) { + auto * hctx = static_cast(dev->context); + delete hctx; + } + delete dev; + } + } +}; + +// Lazily create the context (DSP session + mempool) if it doesn't exist yet. +// The ggml framework calls get_buffer_type / supports_buft BEFORE init_backend +// during model loading, so the context must exist by then. +// Called from get_buffer_type, get_repack_buffer_type, supports_buft, and +// device_init_backend. Context persists across inferences (ggml_backend_hexagon_free +// only deletes the backend, not the context) and is freed during registry shutdown. +static ggml_backend_hexagon_context * ggml_backend_hexagon_ensure_context(ggml_backend_dev_t dev) { + if (nullptr != dev && nullptr != dev->context) { + return (ggml_backend_hexagon_context *)dev->context; + } + + ggmlhexagon_load_cfg(); + ggmlhexagon_check_valid_appcfg(); + + // Find dev_index by matching dev in the registry + int dev_index = 0; + if (nullptr != dev) { + auto * reg_ctx = (ggml_backend_hexagon_reg_context *)g_reg_ctx; + if (reg_ctx) { + for (size_t i = 0; i < reg_ctx->devices.size(); i++) { + if (reg_ctx->devices[i] == dev) { + dev_index = (int)i; + break; + } + } + } + } + if (dev_index >= GGML_HEXAGON_MAX_DEVICES) { + GGMLHEXAGON_LOG_ERROR("invalid dev_index %d", dev_index); + return nullptr; + } + + GGMLHEXAGON_LOG_ALWAYS("creating context for dev_index=%d", dev_index); + ggml_backend_hexagon_context * ctx = nullptr; + try { + ctx = new ggml_backend_hexagon_context(dev_index, dev); + } catch (const std::exception & e) { + GGMLHEXAGON_LOG_ERROR("%s", e.what()); + return nullptr; + } + GGML_ASSERT(0 != ctx->ggmlop_handle); + if (nullptr != dev) { + dev->context = ctx; + } + return ctx; +} + +static ggml_backend_t ggml_backend_hexagon_device_init_backend(ggml_backend_dev_t dev, const char * params) { + ggmlhexagon_load_cfg(); + ggmlhexagon_check_valid_appcfg(); + + // Get the device from registry if not provided + if (nullptr == dev) { + int dev_index = 0; + if (nullptr != params) { + dev_index = (int)(intptr_t)params; + if (dev_index < 0) dev_index = 0; + } + auto * reg_ctx = (ggml_backend_hexagon_reg_context *)g_reg_ctx; + if (reg_ctx && dev_index < (int)reg_ctx->devices.size()) { + dev = reg_ctx->devices[dev_index]; + } + } + + // Ensure context exists (may have been created by get_buffer_type) + auto * ctx = ggml_backend_hexagon_ensure_context(dev); + if (nullptr == ctx) { + GGMLHEXAGON_LOG_ERROR("failed to create context"); + return nullptr; + } + + // If backend already exists for this context, return it + if (nullptr != ctx->backend) { + GGMLHEXAGON_LOG_ALWAYS("backend already exists for device %d, reusing", ctx->device); + return ctx->backend; + } + + ggml_backend_hexagon_interface.graph_optimize = + g_hexagon_appcfg.enable_graph_optimize ? ggml_backend_hexagon_graph_optimize : nullptr; + GGMLHEXAGON_LOG_ALWAYS("graph_optimize: %s", g_hexagon_appcfg.enable_graph_optimize ? "enabled" : "disabled"); + + ggml_backend_t hexagon_backend = new ggml_backend{ + /* .guid = */ ggml_backend_hexagon_guid(), + /* .iface = */ ggml_backend_hexagon_interface, + /* .device = */ dev, + /* .context = */ ctx + }; + + ctx->backend = hexagon_backend; + return hexagon_backend; +} + +static ggml_backend_buffer_type_t ggml_backend_hexagon_device_get_buffer_type(ggml_backend_dev_t dev) { + ggml_backend_hexagon_context * ctx = ggml_backend_hexagon_ensure_context(dev); + if (nullptr == ctx) { + GGMLHEXAGON_LOG_ERROR("get_buffer_type: failed to create context"); + return nullptr; + } + GGMLHEXAGON_LOG_WARN("get_buffer_type: device=%d domain_id=%d buft=%p", ctx->device, ctx->domain_id, (void*)&ctx->buffer_type); + return &ctx->buffer_type; +} + +static ggml_backend_buffer_type_t ggml_backend_hexagon_device_get_repack_buffer_type(ggml_backend_dev_t dev) { + ggml_backend_hexagon_context * ctx = ggml_backend_hexagon_ensure_context(dev); + if (nullptr == ctx) { + GGMLHEXAGON_LOG_ERROR("get_repack_buffer_type: failed to create context"); + return nullptr; + } + return &ctx->repack_buffer_type; +} + +static ggml_backend_buffer_type_t * ggml_backend_hexagon_device_get_extra_buffers_type(ggml_backend_dev_t dev) { + static ggml_backend_buffer_type_t bufts[2]; + bufts[0] = ggml_backend_hexagon_device_get_repack_buffer_type(dev); + bufts[1] = NULL; + return bufts; +} + +static bool ggml_backend_hexagon_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { + if (ggml_backend_buft_is_hexagon(buft) || ggml_backend_buft_is_hexagon_repack(buft)) { + ggml_backend_hexagon_context * dev_ctx = ggml_backend_hexagon_ensure_context(dev); + if (nullptr == dev_ctx) { + GGMLHEXAGON_LOG_ERROR("supports_buft: failed to create context"); + return false; + } + ggml_backend_hexagon_context * buft_ctx = (ggml_backend_hexagon_context *)buft->context; + return buft_ctx->device == dev_ctx->device; + } + return false; +} + +static struct ggml_backend_device_i ggml_backend_hexagon_device_interface = { + /* .get_name = */ ggml_backend_hexagon_device_get_name, + /* .get_description = */ ggml_backend_hexagon_device_get_description, + /* .get_memory = */ ggml_backend_hexagon_device_get_memory, + /* .get_type = */ ggml_backend_hexagon_device_get_type, + /* .get_props = */ ggml_backend_hexagon_device_get_props, + /* .init_backend = */ ggml_backend_hexagon_device_init_backend, + /* .get_buffer_type = */ ggml_backend_hexagon_device_get_buffer_type, + /* .get_host_buffer_type = */ nullptr, + /* .buffer_from_host_ptr = */ nullptr, + /* .supports_op = */ ggmlhexagon_can_handle_op_through_cdsp, + /* .supports_buft = */ ggml_backend_hexagon_device_supports_buft, + /* .offload_op = */ nullptr, + /* .event_new = */ nullptr, + /* .event_free = */ nullptr, + /* .event_synchronize = */ nullptr, +}; + +bool ggml_backend_is_hexagon(ggml_backend_t backend) { + return backend != nullptr && ggml_guid_matches(backend->guid, ggml_backend_hexagon_guid()); +} + +static void ggml_backend_hexagon_set_n_threads(ggml_backend_t backend, int n_threads) { + GGML_ASSERT(ggml_backend_is_hexagon(backend)); + + struct ggml_backend_hexagon_context * ctx = (struct ggml_backend_hexagon_context *)backend->context; + // Prefer DSP-reported cap (set in ggmlhexagon_init_dsp via setclocks + // out param, equals max_hw_threads - 2). Fall back to cfg value if + // set_n_threads is called before DSP init completes. + const int cap = (ctx->dsp_thread_counts > 0) ? ctx->dsp_thread_counts : g_hexagon_appcfg.thread_counts; + int new_threads = (n_threads < cap) ? n_threads : cap; + if (new_threads != ctx->n_threads) { + ctx->n_threads = new_threads; + // VTCM partitioning depends on thread count. Cached mm params and cached + // graph descriptors both embed it, so drop both. + ctx->mm_params_cache.clear(); + ctx->cgraph_cache.clear(); + } +} + +static int ggml_backend_hexagon_get_device_count() { + return g_hexagon_appcfg.ndev; +} + +static void ggml_backend_hexagon_atexit_cleanup() { + if (g_reg_ctx) { + delete g_reg_ctx; + g_reg_ctx = nullptr; + } +} + +static const char * ggml_backend_hexagon_reg_get_name(ggml_backend_reg_t reg) { + GGML_UNUSED(reg); + return "HTP"; +} + +static size_t ggml_backend_hexagon_reg_get_device_count(ggml_backend_reg_t reg) { + ggml_backend_hexagon_reg_context * ctx = (ggml_backend_hexagon_reg_context *)reg->context; + return ctx->devices.size(); +} + +static ggml_backend_dev_t ggml_backend_hexagon_reg_get_device(ggml_backend_reg_t reg, size_t index) { + ggml_backend_hexagon_reg_context * ctx = (ggml_backend_hexagon_reg_context *)reg->context; + GGMLHEXAGON_LOG_WARN("reg_get_device: index=%zu count=%zu", index, ctx->devices.size()); + if (index >= ctx->devices.size()) { + GGMLHEXAGON_LOG_ERROR("invalid device index %d (count=%zu)", index, ctx->devices.size()); + return nullptr; + } + return ctx->devices[index]; +} + +static void * ggml_backend_hexagon_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) { + GGML_UNUSED(reg); + + if (nullptr == name) + return nullptr; + + if (0 == strcmp(name, "ggml_backend_set_n_threads")) { + return (void *)ggml_backend_hexagon_set_n_threads; + } + if (0 == strcmp(name, "ggml_backend_dev_get_extra_bufts")) { + return (void *)ggml_backend_hexagon_device_get_extra_buffers_type; + } + + return nullptr; +} + +static const ggml_backend_reg_i ggml_backend_hexagon_reg_interface = { + /* .get_name = */ ggml_backend_hexagon_reg_get_name, + /* .get_device_count = */ ggml_backend_hexagon_reg_get_device_count, + /* .get_device = */ ggml_backend_hexagon_reg_get_device, + /* .get_proc_address = */ ggml_backend_hexagon_reg_get_proc_address, +}; + +ggml_backend_reg_t ggml_backend_hexagon_reg() { + static ggml_backend_reg reg; + static bool initialized = false; + + ggmlhexagon_load_cfg(); + ggmlhexagon_check_valid_appcfg(); + + { + static std::mutex mutex; + std::lock_guard lock(mutex); + if (!initialized) { + int ret = htpdrv_init(); + if (AEE_SUCCESS != ret) { + GGMLHEXAGON_LOG_ERROR("htpdrv_init failed with error %d", ret); + return nullptr; + } + + int ndev = g_hexagon_appcfg.ndev; + ggml_backend_hexagon_reg_context * ctx = new ggml_backend_hexagon_reg_context; + GGMLHEXAGON_LOG_ALWAYS("registering %d Hexagon device(s), ndev=%d", ndev, g_hexagon_appcfg.ndev); + + for (int i = 0; i < ndev; i++) { + if (i >= GGML_HEXAGON_MAX_DEVICES) { + GGMLHEXAGON_LOG_WARN("ndev=%d exceeds GGML_HEXAGON_MAX_DEVICES=%d, only %d devices registered", + ndev, GGML_HEXAGON_MAX_DEVICES, i); + break; + } + + GGMLHEXAGON_LOG_ALWAYS("register backend device %d (context created lazily)", i); + // Only register the device struct here. Context (DSP session, + // mempool) is created lazily by ggml_backend_hexagon_ensure_context + // (called from get_buffer_type or init_backend) and persists across + // inferences. It is freed during registry shutdown. + ggml_backend_dev_t dev = new ggml_backend_device{ + /* .iface = */ ggml_backend_hexagon_device_interface, + /* .reg = */ ®, + /* .context = */ nullptr // set in device_init_backend + }; + ctx->devices.push_back(dev); + } + + reg = ggml_backend_reg { + /* .api_version = */ GGML_BACKEND_API_VERSION, + /* .iface = */ ggml_backend_hexagon_reg_interface, + /* .context = */ ctx + }; + + g_reg_ctx = ctx; + std::atexit(ggml_backend_hexagon_atexit_cleanup); + } + + initialized = true; + } + return ® +} + +static const char * ggml_backend_hexagon_get_devname(size_t dev_num) { + // CDSP devices: HTP0, HTP1, ... + static char dev_names[GGML_HEXAGON_MAX_DEVICES][32]; + if (dev_num < GGML_HEXAGON_MAX_DEVICES) { + snprintf(dev_names[dev_num], sizeof(dev_names[dev_num]), "HTP%zu", dev_num); + return dev_names[dev_num]; + } + return "unknown"; +} + +static ggml_backend_t ggml_backend_hexagon_init_ext(size_t device, const char * runtime_libpath) { + ggmlhexagon_load_cfg(); + ggmlhexagon_check_valid_appcfg(); + + if (nullptr == runtime_libpath) { + runtime_libpath = g_hexagon_appcfg.runtime_libpath; + } + + GGMLHEXAGON_LOG_ALWAYS("device %d", device); + GGMLHEXAGON_LOG_ALWAYS("runtime libpath %s", runtime_libpath); + if (device >= GGML_HEXAGON_MAX_DEVICES) { + GGMLHEXAGON_LOG_ERROR("invalid device %d", device); + return nullptr; + } + + if (0 != memcmp(runtime_libpath, g_hexagon_appcfg.runtime_libpath, strlen(g_hexagon_appcfg.runtime_libpath))) { + //re-setting runtime libpath + ggmlhexagon_set_runtime_path(device, runtime_libpath); + } + + // Get the device from registry + ggml_backend_dev_t dev = ggml_backend_reg_dev_get(ggml_backend_hexagon_reg(), device); + + // Ensure context exists (lazy creation, same as device_init_backend) + auto * ctx = ggml_backend_hexagon_ensure_context(dev); + if (nullptr == ctx) { + GGMLHEXAGON_LOG_ERROR("failed to create context"); + return nullptr; + } + + // If backend already exists, return it + if (nullptr != ctx->backend) { + GGMLHEXAGON_LOG_ALWAYS("backend already exists for device %d, reusing", ctx->device); + return ctx->backend; + } + + ggml_backend_hexagon_interface.graph_optimize = + g_hexagon_appcfg.enable_graph_optimize ? ggml_backend_hexagon_graph_optimize : nullptr; + GGMLHEXAGON_LOG_ALWAYS("graph_optimize: %s", g_hexagon_appcfg.enable_graph_optimize ? "enabled" : "disabled"); + + ggml_backend_t hexagon_backend = new ggml_backend{ + /* .guid = */ ggml_backend_hexagon_guid(), + /* .iface = */ ggml_backend_hexagon_interface, + /* .device = */ dev, + /* .context = */ ctx + }; + + ctx->backend = hexagon_backend; + return hexagon_backend; +} + +ggml_backend_t ggml_backend_hexagon_init(void) { + return ggml_backend_hexagon_init_ext(0, nullptr); +} + +GGML_BACKEND_DL_IMPL(ggml_backend_hexagon_reg) diff --git a/ggml/src/ggml-hexagon/htp/CMakeLists.txt b/ggml/src/ggml-hexagon/htp/CMakeLists.txt index b00aa2bc94c3..505f1beaf878 100644 --- a/ggml/src/ggml-hexagon/htp/CMakeLists.txt +++ b/ggml/src/ggml-hexagon/htp/CMakeLists.txt @@ -12,15 +12,12 @@ include_directories( ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) -set(HTP_LIB ggml-htp-${DSP_VERSION}) - -add_library(${HTP_LIB} SHARED - main.c - htp_iface_skel.c +# Op sources shared by both skel variants (dspqueue entry via main.c and +# mempool/FastRPC-invoke entry via entry.c). Entry points and glue differ per variant. +set(HTP_OPS_SRCS work-queue.c dma-queue.c hmx-queue.c - htp-tensor.c matmul-ops.c flash-attn-ops.c gated-delta-net-ops.c @@ -45,13 +42,50 @@ add_library(${HTP_LIB} SHARED im2col-ops.c ) -target_compile_definitions(${HTP_LIB} PRIVATE - $,HTP_DEBUG=1,NDEBUG=1> - $,FARF_HIGH=1,>) +# Both variants build ggml-htp-${DSP_VERSION} (libggml-htp-vXX.so); only one +# is built per configuration. +option(GGML_HEXAGON_USE_MEMPOOL "Build the mempool/FastRPC-invoke skel variant (entry.c) instead of the dspqueue variant (main.c)" OFF) + +if (GGML_HEXAGON_USE_MEMPOOL) + set(HTP_LIB ggml-htp-${DSP_VERSION}) + + add_library(${HTP_LIB} SHARED + entry.c + ${CMAKE_CURRENT_BINARY_DIR}/ggml_dsp_skel.c + ${HTP_OPS_SRCS} + ) + + target_compile_definitions(${HTP_LIB} PRIVATE + GGML_HEXAGON_USE_MEMPOOL=1 + $,HTP_DEBUG=1,NDEBUG=1> + $,FARF_HIGH=1,> + ) + + target_compile_options(${HTP_LIB} PRIVATE -Wno-error) + + build_idl(ggml_dsp.idl ${HTP_LIB}) + + set_target_properties(${HTP_LIB} PROPERTIES EXPORT_COMPILE_COMMANDS ON) + + install(TARGETS ${HTP_LIB}) +else() + set(HTP_LIB ggml-htp-${DSP_VERSION}) + + add_library(${HTP_LIB} SHARED + main.c + htp_iface_skel.c + htp-tensor.c + ${HTP_OPS_SRCS} + ) + target_compile_definitions(${HTP_LIB} PRIVATE + $,HTP_DEBUG=1,NDEBUG=1> + $,FARF_HIGH=1,> + ) -build_idl(htp_iface.idl ${HTP_LIB}) + build_idl(htp_iface.idl ${HTP_LIB}) -set_target_properties(${HTP_LIB} PROPERTIES EXPORT_COMPILE_COMMANDS ON) + set_target_properties(${HTP_LIB} PROPERTIES EXPORT_COMPILE_COMMANDS ON) -install(TARGETS ${HTP_LIB}) + install(TARGETS ${HTP_LIB}) +endif() diff --git a/ggml/src/ggml-hexagon/htp/dsp-ctx.h b/ggml/src/ggml-hexagon/htp/dsp-ctx.h new file mode 100644 index 000000000000..46188724832c --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/dsp-ctx.h @@ -0,0 +1,290 @@ +#ifndef GGMLDSP_CTX_H +#define GGMLDSP_CTX_H + +#include +#include +#include +#include + +#include "ggml.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Op-level profiling is disabled by default + * Compile with -DHEX_OP_PROF=1 to enable */ +#ifndef HEX_OP_PROF +#define HEX_OP_PROF 0 +#endif + +/* Alignment requirements */ +#define HEX_BATCH_ALIGN 128 +#define HEX_TENSOR_ALIGN 128 +#define HEX_OP_ALIGN 128 + +// HTP_TENSOR_FLUSHED was removed in upstream b2dd28a3b: per-tensor flush +// flags were replaced by htp_context.dirty_map, maintained by the dspqueue +// scheduler in main.c (kernels never flush L2 themselves). The mempool +// entry.c replaces that scheduler and does its own cache management, so +// htp_tensor.flags is never read on this path. Defined as 0 to keep the +// legacy assignments in entry.c compiling. +#ifndef HTP_TENSOR_FLUSHED +#define HTP_TENSOR_FLUSHED 0 +#endif + +/* Array size limits for per-batch tracking arrays. */ +#ifndef WEIGHT_INVAL_MAX_PTRS +#define WEIGHT_INVAL_MAX_PTRS 4096 +#endif + +#ifndef DSP_OPT_MAX_TENSORS +#define DSP_OPT_MAX_TENSORS 4096 +#endif + +/* max n_ops * HTP_OP_MAX_OUTPUTS; n_ops upper bound = DSP_OPT_MAX_TENSORS * 4 */ +#ifndef DSP_OPT_MAX_BATCH_DSTS +#define DSP_OPT_MAX_BATCH_DSTS (DSP_OPT_MAX_TENSORS * 4 * 4) +#endif + +#ifndef HTP_OP_MAX_OUTPUTS +#define HTP_OP_MAX_OUTPUTS 4 +#endif + +#ifndef HEX_OP_PROF_BUCKETS +#define HEX_OP_PROF_BUCKETS 64 +#endif + +#define GGMLHEXAGON_LOGBUF_LEN 4096 +#define GGMLHEXAGON_TMPBUF_LEN 256 + +#define GGMLHEXAGON_LOG_ALWAYS(...) ggmlhexagon_log_always_internal(GGML_LOG_LEVEL_NONE , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#define GGMLHEXAGON_LOG_ERROR(...) ggmlhexagon_log_always_internal(GGML_LOG_LEVEL_ERROR, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#define GGMLHEXAGON_LOG_VERBOSE(...) ggmlhexagon_log_always_internal(GGML_LOG_LEVEL_CONT , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#define GGMLHEXAGON_LOG_WARN(...) ggmlhexagon_log_internal(GGML_LOG_LEVEL_WARN , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#define GGMLHEXAGON_LOG_INFO(...) ggmlhexagon_log_internal(GGML_LOG_LEVEL_INFO , __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) + +#ifndef NDEBUG +#define GGMLHEXAGON_LOG_DEBUG(...) ggmlhexagon_log_internal(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__) +#else +#define GGMLHEXAGON_LOG_DEBUG(...) +#endif + +// Forward declarations for types used in dsp_context. +// Note: hmx_queue_t is `struct hmx_queue_s *` (typedef in hmx-queue.h), so the +// forward declaration must use the _s suffix to match the new Qualcomm API. +struct hmx_queue_s; +struct htp_context; +struct htp_tensor; + +typedef struct dsptensor dsptensor; + +struct dsptensor { + int32_t type; + int32_t ne[4]; + int32_t nb[4]; + int32_t op; + int32_t op_params[16]; + int32_t flags; + void * data; + int data_len; +}; + +typedef struct { + void * base; + size_t len; + uint32_t tensor_idx; +} dsp_dst_range_t; + +typedef struct dsp_op_desc dsp_op_desc; +struct dsp_op_desc { + int32_t opcode; + int32_t params[16]; + int32_t src0_idx; + int32_t src1_idx; + int32_t src2_idx; + int32_t src3_idx; + int32_t dst_idx; +}; + +/* + * Shared memory batch descriptor for ION-based multi-op offload. + * + * Layout in mempool: + * [hex_batch_hdr] + * [hex_op_desc[0..n_ops-1]] + * [hex_tensor_desc[0..n_tensors-1]] + * + * All data_offset fields are byte offsets from the mempool base. + * DSP side accesses data as: mempool_dsp_base + tensor->data_offset + */ + +/* Tensor descriptor - uses offset instead of pointer */ +typedef struct hex_tensor_desc { + int32_t type; /* ggml_type */ + int32_t ne[4]; /* element counts per dimension */ + int32_t nb[4]; /* strides (bytes) per dimension */ + int32_t op_params[16]; /* operation-specific parameters */ + uint32_t flags; /* 0=ION tensor, 1=mirrored (heap), 2=weight (skip cache flush) */ + uint32_t data_offset; /* byte offset of data in mempool */ + uint32_t data_len; /* data length in bytes */ +} hex_tensor_desc; + +/* Op descriptor - references tensors by index. + * Mirrors Qualcomm htp_op_desc layout: src_idx[HTP_OP_MAX_INPUTS=6] + dst_idx[HTP_OP_MAX_OUTPUTS=4]. */ +typedef struct hex_op_desc { + int32_t opcode; /* GGML_OP_XXX */ + int32_t params[16]; /* operation parameters */ + int32_t kernel_params[32]; /* precomputed kernel params (e.g. htp_mm_kernel_params for MUL_MAT) */ + int32_t src_idx[6]; /* indices into tensor table (-1 = none); mirrors htp_op_desc.src[6] */ + int32_t dst_idx[4]; /* multi-output support (e.g. QKV fusion), -1 = unused */ + int32_t htp_opcode; /* Direct HTP opcode for fused ops (0 = use ggml_op_to_htp_op) */ +} hex_op_desc; + +/* Batch header - entry point for DSP to find everything */ +typedef struct hex_batch_hdr { + uint32_t n_ops; /* number of ops */ + uint32_t n_tensors; /* number of tensors */ + uint32_t ops_offset; /* offset from hdr start -> hex_op_desc[] */ + uint32_t tensors_offset; /* offset from hdr start -> hex_tensor_desc[] */ + uint32_t total_size; /* total size of this batch region (hdr + ops + tensors) */ + uint32_t reserved; /* padding / future use */ +} hex_batch_hdr; + +// DSP session context: bundles all per-session state. +// Allocated in ggml_dsp_open, freed in ggml_dsp_close. +struct dsp_context { + // Configuration + int thread_counts; + int max_hw_threads; /* qurt_sysenv_get_max_hw_threads, set once at open */ + int dump_diag_info; + + // VTCM + void * vtcm_base; + size_t vtcm_size; + unsigned int compute_res_ctx_id; + volatile int vtcm_needs_release; + volatile int vtcm_valid; + int thread_prio; + + // Power + int power_ctx; + void * hexagon_power_ctx; + + // HMX + int hmx_available; + struct hmx_queue_s * hmx_queue; + // Backing buffer for hmx_queue (NULL if hmx_queue is owned externally). + // Allocated via memalign in ggml_dsp_setclocks and freed in ggml_dsp_close. + void * hmx_queue_buf; + + // mempool + void * mempool_dsp_base; + size_t mempool_dsp_size; + + // DSP-side entry.c cache optimization bitmask. Pushed by AP at init via + // execute_batch(0xFFFC) special mode (no IDL change). All three bits are + // are wired into ggml_dsp_execute_batch(); dsp_cache_mode=0 is behaviorally + // identical to baseline 29c1cf196. + // bit 0 (0x1): first-touch weight bitmap - skip dcinva for repack weights (flags==2) after first access + // bit 1 (0x2): skip dcinva for prior dst - DSP's own dst writes stay in L2; next op's src read skips dcinva + // bit 2 (0x4): bulk dst flush at batch end - collect/sort/merge dst ranges, flush once per region + // bit 3..31 : reserved for future use + uint32_t dsp_cache_mode; + + // DSP-side bit 0 (first-touch weight bitmap) trace enable. Pushed by AP at + // init via the same execute_batch(0xFFFC) special mode as dsp_cache_mode + // (bit 16 of the same payload word, so the special-mode encoding is + // payload = (dsp_cache_trace_bit0 << 16) | (dsp_cache_mode & 0x7u) + // ). When non-zero, INVAL_SRC_IF_NEEDED emits one [DSP-CACHE-TRACE-BIT0] + // log line per bit 0 decision (SKIP or INVAL), with op index, src index, + // weight address, weight length, current ctx id, and qurt_timer tick count. + // Default 0 (off) so production perf is unaffected. Set to 1 only when + // diagnosing the bit 0 stale L2 read bug (llama3 33% prompt-repeat rate + // observed 2026-07-10). Once the bug is root-caused this can be removed. + uint32_t dsp_cache_trace_bit0; + + // DSP-side bit 1 (skip dcinva for prior dst) trace enable. Pushed by AP at + // init via bit 17 of the same execute_batch(0xFFFC) payload word, so the + // special-mode encoding is + // payload = (dsp_cache_trace_bit1 << 17) | (dsp_cache_trace_bit0 << 16) + // | (dsp_cache_mode & 0x7u) + // When non-zero, INVAL_SRC_IF_NEEDED emits one [DSP-CACHE-TRACE-BIT1] log + // line per bit 1 decision (SKIP if prior_dst_contains_src, INVAL otherwise) + // with the same op/src/ptr/len fields as the bit 0 trace. Default 0 (off) + // so production perf is unaffected. Set to 1 to measure bit 1 SKIP rate + // (how often INVAL_SRC_IF_NEEDED takes the prior-dst skip path). Currently + // a no-op: PRIOR_DST_MAX_LEN=64 (entry.c) excludes all real-world cgraph + // intermediate tensors (>= 256B). Pair with dsp_cache_trace_bit0 to + // cross-check L2 staleness between weight and activation domains. + uint32_t dsp_cache_trace_bit1; + + // htp_context for calling the shared execute_op. + struct htp_context * htp_ctx; + + // Backing buffers for queues owned by this dsp_context (allocated via + // memalign in ggml_dsp_setclocks, freed in ggml_dsp_close). The new + // Qualcomm API (b2dd28a3b) requires callers to provide pre-allocated + // memory to *_queue_init and does not free it in *_queue_free, so we + // must track these buffers separately to avoid leaking them. + // work_queue_buf : backing for htp_ctx->work_queue + // dma_queue_bufs[i] : backing for htp_ctx->dma_cached[i] (NULL when slot unused) + // dma_alias_bufs[i] : backing for htp_ctx->dma[i] alias (NULL when slot unused) + void * work_queue_buf; + void * dma_queue_bufs[16]; // HTP_MAX_NTHREADS == 10, but use 16 for safety + void * dma_alias_bufs[16]; + + // Per-session state (moved from file-static globals for multi-session isolation). + // Small arrays are embedded; large arrays are allocated from arrays_pool. + + // Weight first-touch invalidate tracking (bit 0) + const void * weight_inval_ptrs[WEIGHT_INVAL_MAX_PTRS]; + uint32_t weight_inval_count; + + // Per-batch src invalidation tracking + uint8_t batch_tensor_needs_inval[DSP_OPT_MAX_TENSORS]; + + // bit 3 last consumer op index per tensor + uint32_t tensor_last_use_op[DSP_OPT_MAX_TENSORS]; + + // Per-op dst staging buffers + dsptensor dst_dt_buf [HTP_OP_MAX_OUTPUTS]; + const dsptensor * dst_dt_ptrs[HTP_OP_MAX_OUTPUTS]; + + // Large arrays: single mempool allocation for cache locality + void * arrays_pool; + dsptensor * pre_dt; // [DSP_OPT_MAX_TENSORS] + struct htp_tensor * pre_ht; // [DSP_OPT_MAX_TENSORS] + dsp_dst_range_t * prior_dst_ranges; // [DSP_OPT_MAX_BATCH_DSTS] + dsp_dst_range_t * bulk_flush_ranges; // [DSP_OPT_MAX_BATCH_DSTS] + int prior_dst_count; + int bulk_flush_count; + +#if HEX_OP_PROF + // Per-op profiling (compiled in when HEX_OP_PROF is non-zero) + uint64_t op_prof_dur_us[HEX_OP_PROF_BUCKETS]; + uint64_t op_prof_count [HEX_OP_PROF_BUCKETS]; + uint64_t op_prof_min_us[HEX_OP_PROF_BUCKETS]; + uint64_t op_prof_max_us[HEX_OP_PROF_BUCKETS]; + uint32_t op_prof_batch_count; + uint64_t op_prof_batch_wall_us; + uint64_t nonop_hdr_inval_us; + uint64_t nonop_preconvert_us; + uint64_t nonop_w_inval_us; + uint64_t nonop_w_inval_bytes; + uint64_t nonop_a_inval_us; + uint64_t nonop_a_inval_bytes; + uint64_t nonop_dst_track_us; + uint64_t nonop_bulk_flush_us; + uint64_t nonop_queue_us; +#endif +}; + +void ggmlhexagon_log_internal(int level, const char * file, const char * func, int line, const char * format, ...); +void ggmlhexagon_log_always_internal(int level, const char * file, const char * func, int line, const char * format, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* GGMLDSP_CTX_H */ diff --git a/ggml/src/ggml-hexagon/htp/entry.c b/ggml/src/ggml-hexagon/htp/entry.c new file mode 100644 index 000000000000..bbe6ddf3549b --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/entry.c @@ -0,0 +1,2332 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ggml.h" + +#include "dsp-ctx.h" +#include "hmx-queue.h" +#include "htp-ctx.h" +#include "matmul-ops.h" +#include "flash-attn-ops.h" + +// ================================================================================================= +// forward declarations, global vars, macros +// ================================================================================================= +#define DSP_CACHE_LINE_SIZE 64 + +#define DEFAULT_VTCM_SIZE (8 * 1024 * 1024) + +#define HEX_OP_PROF_DUMP_INTERVAL 25 + +// Queue capacity/stack sizes: mirror htp/main.c +#define HMX_QUEUE_CAPACITY 16 +#define HMX_QUEUE_STACK_SIZE 16384 +#define WORK_QUEUE_CAPACITY 16 +#define WORK_QUEUE_STACK_SIZE 16384 + +// Max dst len for prior-dst skip (bit 1): single cacheline only. +// Larger ranges risk stale L2 reads from async DMA/HMX paths. +#define PRIOR_DST_MAX_LEN DSP_CACHE_LINE_SIZE + +#define INVAL_SRC_IF_NEEDED(op_i, src_idx, dt_ptr, tensor_idx) do { \ + if (dt_ptr) { \ + /* Per-batch dedup: skip if already invalidated and not dirtied since. */ \ + if (!g_dsp_ctx->batch_tensor_needs_inval[tensor_idx]) { \ + /* already invalidated this batch, L2 line is fresh */ \ + } else if ((dt_ptr)->flags & 0x2) { \ + /* Weight tensor: bit 0 check */ \ + bool _already_inval = false; \ + if ((g_dsp_ctx->dsp_cache_mode & 0x1) && \ + (_already_inval = weight_inval_check_and_mark((dt_ptr)->data))) { \ + if (g_dsp_ctx->dsp_cache_trace_bit0) { \ + GGMLHEXAGON_LOG_INFO("[DSP-CACHE-TRACE-BIT0] op=%u src=%d SKIP ptr=%p len=0x%x (cache_mode=0x%x)", \ + (op_i), (int)(src_idx), (dt_ptr)->data, (dt_ptr)->data_len, g_dsp_ctx->dsp_cache_mode); \ + } \ + } else { \ + prof_cache_inval_range((dt_ptr)->data, (dt_ptr)->data_len, 1); \ + if (g_dsp_ctx->dsp_cache_trace_bit0) { \ + GGMLHEXAGON_LOG_INFO("[DSP-CACHE-TRACE-BIT0] op=%u src=%d INVAL ptr=%p len=0x%x (cache_mode=0x%x)", \ + (op_i), (int)(src_idx), (dt_ptr)->data, (dt_ptr)->data_len, g_dsp_ctx->dsp_cache_mode); \ + } \ + } \ + } else { \ + /* Activation tensor: bit 1 check */ \ + if ((g_dsp_ctx->dsp_cache_mode & 0x2) && \ + prior_dst_contains_src((tensor_idx), (dt_ptr)->data, (dt_ptr)->data_len)) { \ + if (g_dsp_ctx->dsp_cache_trace_bit1) { \ + GGMLHEXAGON_LOG_INFO("[DSP-CACHE-TRACE-BIT1] op=%u src=%d SKIP ptr=%p len=0x%x (cache_mode=0x%x)", \ + (op_i), (int)(src_idx), (dt_ptr)->data, (dt_ptr)->data_len, g_dsp_ctx->dsp_cache_mode); \ + } \ + } else { \ + prof_cache_inval_range((dt_ptr)->data, (dt_ptr)->data_len, 0); \ + if (g_dsp_ctx->dsp_cache_trace_bit1) { \ + GGMLHEXAGON_LOG_INFO("[DSP-CACHE-TRACE-BIT1] op=%u src=%d INVAL ptr=%p len=0x%x (cache_mode=0x%x)", \ + (op_i), (int)(src_idx), (dt_ptr)->data, (dt_ptr)->data_len, g_dsp_ctx->dsp_cache_mode); \ + } \ + } \ + } \ + g_dsp_ctx->batch_tensor_needs_inval[tensor_idx] = 0; \ + } \ +} while (0) + +static struct dsp_context * g_dsp_ctx = NULL; + +// ================================================================================================= +// troubleshooting and profiler +// ================================================================================================= +// ggml_abort() is declared in ggml.h; GGML_ASSERT/GGML_ABORT macros call it. +void ggml_abort(const char * file, int line, const char * fmt, ...) { + abort(); +} + +int64_t ggml_time_us(void) { + unsigned long long count; + asm volatile(" %0 = c31:30 " : "=r"(count)); + return (uint64_t)(count) * 10ull / 192ull; +} + +void ggmlhexagon_log_internal(int level, const char *file, const char *func, int line, const char *format, ...) { + if (!g_dsp_ctx || !g_dsp_ctx->dump_diag_info) { + return; + } + GGML_UNUSED(level); + GGML_UNUSED(file); + static char s_ggmlhexagon_log_internal_buf[GGMLHEXAGON_LOGBUF_LEN]; + va_list args; + va_start(args, format); + int len_prefix = snprintf(s_ggmlhexagon_log_internal_buf, GGMLHEXAGON_LOGBUF_LEN, "[%s, %d]: ", func, line); + if (len_prefix < 0 || (size_t)len_prefix >= GGMLHEXAGON_LOGBUF_LEN) { + va_end(args); + return; + } + int len = vsnprintf(s_ggmlhexagon_log_internal_buf + len_prefix, GGMLHEXAGON_LOGBUF_LEN - len_prefix, format, args); + if (len >= 0 && len < (GGMLHEXAGON_LOGBUF_LEN - len_prefix)) { + FARF(ALWAYS, "%s\n", s_ggmlhexagon_log_internal_buf); + } + va_end(args); +} + +void ggmlhexagon_log_always_internal(int level, const char *file, const char *func, int line, const char *format, ...) { + GGML_UNUSED(level); + GGML_UNUSED(file); + static char s_ggmlhexagon_log_buf[GGMLHEXAGON_LOGBUF_LEN]; + va_list args; + va_start(args, format); + int len_prefix = snprintf(s_ggmlhexagon_log_buf, GGMLHEXAGON_LOGBUF_LEN, "[%s, %d]: ", func, line); + if (len_prefix < 0 || (size_t)len_prefix >= GGMLHEXAGON_LOGBUF_LEN) { + va_end(args); + return; + } + int len = vsnprintf(s_ggmlhexagon_log_buf + len_prefix, GGMLHEXAGON_LOGBUF_LEN - len_prefix, format, args); + if (len >= 0 && len < (GGMLHEXAGON_LOGBUF_LEN - len_prefix)) { + FARF(ALWAYS, "%s\n", s_ggmlhexagon_log_buf); + } + va_end(args); +} + +#if HEX_OP_PROF +static const char * htp_op_short_name(unsigned int op) { + switch (op) { + case HTP_OP_MUL_MAT: return "MUL_MAT"; + case HTP_OP_MUL_MAT_ADD: return "MUL_MAT_ADD"; + case HTP_OP_MUL_MAT_ID: return "MUL_MAT_ID"; + case HTP_OP_MUL_MAT_QKV: return "MUL_MAT_QKV"; + case HTP_OP_MUL_MAT_FFN: return "MUL_MAT_FFN"; + case HTP_OP_MUL: return "MUL"; + case HTP_OP_ADD: return "ADD"; + case HTP_OP_SUB: return "SUB"; + case HTP_OP_DIV: return "DIV"; + case HTP_OP_ADD_ID: return "ADD_ID"; + case HTP_OP_NORM: return "NORM"; + case HTP_OP_RMS_NORM: return "RMS_NORM"; + case HTP_OP_RMS_NORM_MUL: return "RMS_NORM_MUL"; + case HTP_OP_SCALE: return "SCALE"; + case HTP_OP_SQR: return "SQR"; + case HTP_OP_SQRT: return "SQRT"; + case HTP_OP_L2_NORM: return "L2_NORM"; + case HTP_OP_UNARY_SOFTPLUS: return "UNARY_SOFTPLUS"; + case HTP_OP_UNARY_SIGMOID: return "UNARY_SIGMOID"; + case HTP_OP_UNARY_NEG: return "UNARY_NEG"; + case HTP_OP_UNARY_EXP: return "UNARY_EXP"; + case HTP_OP_UNARY_TANH: return "UNARY_TANH"; + case HTP_OP_UNARY_SILU: return "UNARY_SILU"; + case HTP_OP_UNARY_GELU: return "UNARY_GELU"; + case HTP_OP_GLU_SWIGLU: return "GLU_SWIGLU"; + case HTP_OP_GLU_SWIGLU_OAI: return "GLU_SWIGLU_OAI"; + case HTP_OP_GLU_GEGLU: return "GLU_GEGLU"; + case HTP_OP_SOFTMAX: return "SOFTMAX"; + case HTP_OP_ROPE: return "ROPE"; + case HTP_OP_FLASH_ATTN_EXT: return "FLASH_ATTN_EXT"; + case HTP_OP_SET_ROWS: return "SET_ROWS"; + case HTP_OP_GET_ROWS: return "GET_ROWS"; + case HTP_OP_SUM_ROWS: return "SUM_ROWS"; + case HTP_OP_CPY: return "CPY"; + case HTP_OP_REPEAT: return "REPEAT"; + case HTP_OP_ARGSORT: return "ARGSORT"; + case HTP_OP_SSM_CONV: return "SSM_CONV"; + case HTP_OP_CUMSUM: return "CUMSUM"; + case HTP_OP_FILL: return "FILL"; + case HTP_OP_DIAG: return "DIAG"; + case HTP_OP_SOLVE_TRI: return "SOLVE_TRI"; + case HTP_OP_PAD: return "PAD"; + case HTP_OP_CONCAT: return "CONCAT"; + case HTP_OP_CLAMP: return "CLAMP"; + case HTP_OP_IM2COL: return "IM2COL"; + case HTP_OP_GATED_DELTA_NET: return "GATED_DELTA_NET"; + case HTP_OP_TRI: return "TRI"; + case HTP_OP_INVALID: return "INVALID"; + default: return NULL; + } +} + +// op_prof_min_us is initialized to UINT64_MAX in ggml_dsp_open. +static void dump_op_prof(const char * tag) { + for (unsigned int i = 0; i < HEX_OP_PROF_BUCKETS; i++) { + if (g_dsp_ctx->op_prof_count[i] == 0) continue; + const char * name = htp_op_short_name(i); + const uint64_t avg = g_dsp_ctx->op_prof_dur_us[i] / g_dsp_ctx->op_prof_count[i]; + // Pre-format numeric fields via snprintf so the field width is honored + // (Hexagon FARF does not implement the width modifier in %9llu, so the + // values would print left-justified otherwise). Widths leave headroom: + // cum -> 10 chars (up to 9_999_999_999 us ~ 2.7h of DSP time) + // count -> 7 chars (up to 9_999_999 calls) + // avg -> 5 chars (per-op cost is bounded by graph structure) + // min -> 5 chars + // max -> 6 chars (handles up to 999_999 us, well above any + // realistic per-op stall in this profiler) + char cum_s[24], cnt_s[24], avg_s[24], min_s[24], max_s[24]; + snprintf(cum_s, sizeof(cum_s), "%10llu", (unsigned long long)g_dsp_ctx->op_prof_dur_us[i]); + snprintf(cnt_s, sizeof(cnt_s), "%7llu", (unsigned long long)g_dsp_ctx->op_prof_count[i]); + snprintf(avg_s, sizeof(avg_s), "%5llu", (unsigned long long)avg); + snprintf(min_s, sizeof(min_s), "%5llu", (unsigned long long)g_dsp_ctx->op_prof_min_us[i]); + snprintf(max_s, sizeof(max_s), "%6llu", (unsigned long long)g_dsp_ctx->op_prof_max_us[i]); + if (name) { + FARF(ERROR, "[OP-PROF] %s op=%s cum=%s us count=%s avg=%s min=%s max=%s us", + tag, name, cum_s, cnt_s, avg_s, min_s, max_s); + } else { + FARF(ERROR, "[OP-PROF] %s op=%u cum=%s us count=%s avg=%s min=%s max=%s us", + tag, i, cum_s, cnt_s, avg_s, min_s, max_s); + } + } +} +#endif // HEX_OP_PROF + +// ================================================================================================= +// cache infrastructure +// ================================================================================================= +static void ggml_dsp_cache_flush_range(void * addr, size_t size) { + if (!addr || size == 0) return; + char * p = (char *)addr; + char * end = p + size; + const size_t line_size = DSP_CACHE_LINE_SIZE; + p = (char *)((uintptr_t)p & ~(line_size - 1)); + for (; p + line_size * 8 <= end; p += line_size * 8) { + Q6_dccleaninva_A(p + line_size * 0); + Q6_dccleaninva_A(p + line_size * 1); + Q6_dccleaninva_A(p + line_size * 2); + Q6_dccleaninva_A(p + line_size * 3); + Q6_dccleaninva_A(p + line_size * 4); + Q6_dccleaninva_A(p + line_size * 5); + Q6_dccleaninva_A(p + line_size * 6); + Q6_dccleaninva_A(p + line_size * 7); + } + for (; p < end; p += line_size) { + Q6_dccleaninva_A(p); + } + __asm__ __volatile__("syncht\n" ::: "memory"); +} + +// Flush range WITHOUT the trailing syncht. Used by bulk_flush_all to issue a +// single syncht after processing all merged regions, instead of one per +// region. Caller must issue syncht before any read of the flushed data. +static inline void ggml_dsp_cache_flush_range_nosync(void * addr, size_t size) { + if (!addr || size == 0) return; + char * p = (char *)addr; + char * end = p + size; + const size_t line_size = DSP_CACHE_LINE_SIZE; + p = (char *)((uintptr_t)p & ~(line_size - 1)); + for (; p + line_size * 8 <= end; p += line_size * 8) { + Q6_dccleaninva_A(p + line_size * 0); + Q6_dccleaninva_A(p + line_size * 1); + Q6_dccleaninva_A(p + line_size * 2); + Q6_dccleaninva_A(p + line_size * 3); + Q6_dccleaninva_A(p + line_size * 4); + Q6_dccleaninva_A(p + line_size * 5); + Q6_dccleaninva_A(p + line_size * 6); + Q6_dccleaninva_A(p + line_size * 7); + } + for (; p < end; p += line_size) { + Q6_dccleaninva_A(p); + } + __asm__ __volatile__("" ::: "memory"); // compiler barrier between cache ops +} + +static void ggml_dsp_cache_inval_range(void * addr, size_t size) { + if (!addr || size == 0) return; + char * p = (char *)addr; + char * end = p + size; + const size_t line_size = DSP_CACHE_LINE_SIZE; + p = (char *)((uintptr_t)p & ~(line_size - 1)); + for (; p + line_size * 8 <= end; p += line_size * 8) { + Q6_dcinva_A(p + line_size * 0); + Q6_dcinva_A(p + line_size * 1); + Q6_dcinva_A(p + line_size * 2); + Q6_dcinva_A(p + line_size * 3); + Q6_dcinva_A(p + line_size * 4); + Q6_dcinva_A(p + line_size * 5); + Q6_dcinva_A(p + line_size * 6); + Q6_dcinva_A(p + line_size * 7); + } + for (; p < end; p += line_size) { + Q6_dcinva_A(p); + } + __asm__ __volatile__("syncht\n" ::: "memory"); +} + +static inline bool weight_inval_check_and_mark(const void * ptr) { + int lo = 0; + int hi = (int)g_dsp_ctx->weight_inval_count - 1; + while (lo <= hi) { + int mid = (lo + hi) >> 1; + const void * mid_ptr = g_dsp_ctx->weight_inval_ptrs[mid]; + if (mid_ptr == ptr) { + return true; // already invalidated, can skip + } + if (mid_ptr < ptr) { + lo = mid + 1; + } else { + hi = mid - 1; + } + } + // Not found: insert at lo and keep array sorted. + if (g_dsp_ctx->weight_inval_count >= WEIGHT_INVAL_MAX_PTRS) { + // Table full: fall back to always invalidate. This is safe but + // loses the optimization. With 4096 slots and read-only weights, + // overflow should not happen in practice. + return false; + } + for (int i = (int)g_dsp_ctx->weight_inval_count; i > lo; i--) { + g_dsp_ctx->weight_inval_ptrs[i] = g_dsp_ctx->weight_inval_ptrs[i - 1]; + } + g_dsp_ctx->weight_inval_ptrs[lo] = ptr; + g_dsp_ctx->weight_inval_count++; + return false; +} + +static inline void weight_inval_reset_all(void) { + g_dsp_ctx->weight_inval_count = 0; +} + +/* A tensor written as dst is no longer read-only: drop it from the + * first-touch weight list so its next src use re-invalidates. Fixes + * cross-graph staleness where a tensor is dst in one (sub)graph and + * misclassified as weight (flags=2) in another, e.g. qwen3-mtp. */ +static inline void weight_inval_unmark(const void * ptr) { + int lo = 0; + int hi = (int)g_dsp_ctx->weight_inval_count - 1; + while (lo <= hi) { + int mid = (lo + hi) >> 1; + const void * mid_ptr = g_dsp_ctx->weight_inval_ptrs[mid]; + if (mid_ptr == ptr) { + for (int i = mid; i < (int)g_dsp_ctx->weight_inval_count - 1; i++) { + g_dsp_ctx->weight_inval_ptrs[i] = g_dsp_ctx->weight_inval_ptrs[i + 1]; + } + g_dsp_ctx->weight_inval_count--; + return; + } + if (mid_ptr < ptr) { + lo = mid + 1; + } else { + hi = mid - 1; + } + } +} + +/* DSP-side cache optimization: prior_dst tracking (bit 1) + bulk flush (bit 2). + * + * Both are gated by g_dsp_ctx->dsp_cache_mode and operate on per-batch state. + * + * - bit 1 (skip dcinva for prior dst): when the next op's src is the *same* + * tensor as a small dst of an earlier op in the same batch, and its range is + * fully contained, skip dcinva because the L2 line is fresh (DSP's own write). + * We require tensor_idx equality and len <= PRIOR_DST_MAX_LEN to avoid stale + * reads across DMA/HMX cache domains or aliased views. The per-op dst tracker + * populates the prior_dst list only when bit 2 is on, so bit 1 alone is a + * no-op (a cfg comment in ggml-hexagon.cfg covers this). + * - bit 2 (bulk dst flush at batch end): collect all dst ranges during the + * op loop, sort + merge adjacent/overlapping ranges at batch end, then + * call ggml_dsp_cache_flush_range() once per merged region. Replaces + * per-op flush with fewer but larger flushes. prior_dst is also updated + * when bit 2 is on, regardless of bit 1. + * + * State is reset at the start of every batch (next batch sees empty lists). + * Both lists are sized for the worst case: DSP_OPT_MAX_TENSORS * 4 ops/batch + * * HTP_OP_MAX_OUTPUTS dst/op (= DSP_OPT_MAX_BATCH_DSTS entries). + * + * Hexagon FastRPC domain processes calls one at a time; g_dsp_ctx is set + * from the handle parameter at every entry point, so per-session state is + * correctly isolated even across multiple open sessions. + * + * bit 0 (first-touch weight tracking) is a separate mechanism; see + * weight_inval_check_and_mark() and INVAL_SRC_IF_NEEDED(). It is session- + * scoped and reset on model reload (via execute_batch 0xFFFC bit 4) so + * new weights at reused ION addresses are properly invalidated. + */ +static inline bool dsp_range_contains(const void * r_base, size_t r_len, + const void * q_base, size_t q_len) { + if (!r_base || !q_base || q_len == 0) return false; + uintptr_t rb = (uintptr_t)r_base; + uintptr_t re = rb + r_len; + uintptr_t qb = (uintptr_t)q_base; + uintptr_t qe = qb + q_len; + return (qb >= rb) && (qe <= re); +} + +static inline bool prior_dst_contains_src(uint32_t src_idx, + const void * base, size_t len) { + if (!base) return false; + for (int i = 0; i < g_dsp_ctx->prior_dst_count; i++) { + if (g_dsp_ctx->prior_dst_ranges[i].tensor_idx == src_idx && + dsp_range_contains(g_dsp_ctx->prior_dst_ranges[i].base, g_dsp_ctx->prior_dst_ranges[i].len, + base, len)) { + return true; + } + } + return false; +} + +static inline void prior_dst_add(void * base, size_t len, uint32_t tensor_idx, + enum htp_op_code op) { + (void)op; + if (!base || len == 0) return; + /* With PRIOR_DST_MAX_LEN == cacheline size, the prior-dst is small enough + * that it should still reside in scalar L2. */ + if (len > PRIOR_DST_MAX_LEN) return; + if (g_dsp_ctx->prior_dst_count >= DSP_OPT_MAX_BATCH_DSTS) return; /* overflow guard */ + g_dsp_ctx->prior_dst_ranges[g_dsp_ctx->prior_dst_count].base = base; + g_dsp_ctx->prior_dst_ranges[g_dsp_ctx->prior_dst_count].len = len; + g_dsp_ctx->prior_dst_ranges[g_dsp_ctx->prior_dst_count].tensor_idx = tensor_idx; + g_dsp_ctx->prior_dst_count++; +} + +static inline void bulk_flush_add(void * base, size_t len) { + if (!base || len == 0) return; + if (g_dsp_ctx->bulk_flush_count >= DSP_OPT_MAX_BATCH_DSTS) { + /* Overflow: fall back to immediate per-range flush for THIS dst only + * (degraded perf, but correctness preserved). */ + ggml_dsp_cache_flush_range(base, len); + return; + } + g_dsp_ctx->bulk_flush_ranges[g_dsp_ctx->bulk_flush_count].base = base; + g_dsp_ctx->bulk_flush_ranges[g_dsp_ctx->bulk_flush_count].len = len; + g_dsp_ctx->bulk_flush_count++; +} + +static inline void prior_dst_reset_all(void) { + g_dsp_ctx->prior_dst_count = 0; +} + +static inline void bulk_flush_reset_all(void) { + g_dsp_ctx->bulk_flush_count = 0; +} + +/* Insertion sort (small N typical for batch dst list: 30-50 dsts). + * Avoids libc qsort dependency and works on 32-bit pointers. */ +static inline void bulk_flush_sort(void) { + for (int i = 1; i < g_dsp_ctx->bulk_flush_count; i++) { + dsp_dst_range_t cur = g_dsp_ctx->bulk_flush_ranges[i]; + int j = i - 1; + while (j >= 0 && + (uintptr_t)g_dsp_ctx->bulk_flush_ranges[j].base > (uintptr_t)cur.base) { + g_dsp_ctx->bulk_flush_ranges[j + 1] = g_dsp_ctx->bulk_flush_ranges[j]; + j--; + } + g_dsp_ctx->bulk_flush_ranges[j + 1] = cur; + } +} + +/* Walk sorted list, merge adjacent/overlapping ranges, flush each merged + * region once. Overlap defined as next_start <= cur_end (1B threshold: + * touching ranges merge). Flushes in sorted order to preserve ION ordering. + * + * Uses ggml_dsp_cache_flush_range_nosync() to avoid one syncht per region; + * a single syncht is issued at the end so AP reads of the flushed tensors + * see fresh DRAM. This batch-end consolidation is the only safe place to + * merge syncht barriers; per-op synchts are still required for the + * bit-2-disabled fallback path (see ggml_dsp_execute_batch). */ +static inline void bulk_flush_all(void) { + if (g_dsp_ctx->bulk_flush_count == 0) return; + bulk_flush_sort(); + void * cur_base = g_dsp_ctx->bulk_flush_ranges[0].base; + uintptr_t cur_end = (uintptr_t)cur_base + g_dsp_ctx->bulk_flush_ranges[0].len; + int i = 1; + while (i < g_dsp_ctx->bulk_flush_count) { + void * next_base = g_dsp_ctx->bulk_flush_ranges[i].base; + uintptr_t next_end = (uintptr_t)next_base + g_dsp_ctx->bulk_flush_ranges[i].len; + if ((uintptr_t)next_base <= cur_end) { + /* Overlap or adjacent: extend current region */ + if (next_end > cur_end) cur_end = next_end; + } else { + /* Gap: flush [cur_base, cur_end) and start new region */ + ggml_dsp_cache_flush_range_nosync(cur_base, (size_t)(cur_end - (uintptr_t)cur_base)); + cur_base = next_base; + cur_end = next_end; + } + i++; + } + ggml_dsp_cache_flush_range_nosync(cur_base, (size_t)(cur_end - (uintptr_t)cur_base)); + /* Single syncht covers all merged-region flushes above. AP reads + * following this point see consistent DRAM. */ + __asm__ __volatile__("syncht\n" ::: "memory"); +} + +#if HEX_OP_PROF +static inline void prof_cache_inval_range(void * p, size_t len, int is_weight) { + const int64_t t0 = ggml_time_us(); + ggml_dsp_cache_inval_range(p, len); + const uint64_t dt = (uint64_t)(ggml_time_us() - t0); + if (is_weight) { + g_dsp_ctx->nonop_w_inval_us += dt; + g_dsp_ctx->nonop_w_inval_bytes += len; + } else { + g_dsp_ctx->nonop_a_inval_us += dt; + g_dsp_ctx->nonop_a_inval_bytes += len; + } +} +#else +#define prof_cache_inval_range(p, len, is_weight) ggml_dsp_cache_inval_range((p), (len)) +#endif // HEX_OP_PROF + +// ================================================================================================= +// VTCM +// ================================================================================================= +static void dsp_vtcm_release(void) { + if (g_dsp_ctx->vtcm_valid) { + g_dsp_ctx->vtcm_valid = 0; + g_dsp_ctx->vtcm_needs_release = 0; + HAP_compute_res_release_cached(g_dsp_ctx->compute_res_ctx_id); + } +} + +static void dsp_vtcm_acquire(void) { + if (!g_dsp_ctx->vtcm_valid) { + int err = HAP_compute_res_acquire_cached(g_dsp_ctx->compute_res_ctx_id, 1000000u); + if (err != 0) { + FARF(ERROR, "failed to acquire VTCM: 0x%08x", (unsigned)err); + abort(); + } + g_dsp_ctx->vtcm_needs_release = 0; + g_dsp_ctx->vtcm_valid = 1; + HAP_compute_res_update_priority(g_dsp_ctx->compute_res_ctx_id, + g_dsp_ctx->thread_prio + 10); + } +} + +static int vtcm_release_callback(unsigned int rctx, void * state) { + // Async notification only: flag that another session wants VTCM. + // Do NOT clear g_dsp_ctx->vtcm_valid here - the current batch keeps running + // and releases VTCM at the batch boundary. + g_dsp_ctx->vtcm_needs_release = 1; + return 0; +} + +// ================================================================================================= +// IDL helper functions +// ================================================================================================= +static int power_on_hvx_hmx(struct dsp_context * ctx) { + HAP_power_request_t req; + + /* Note: ctx->power_ctx is left NULL (calloc) and passed as the context + * argument; HAP_power_set treats it as reserved on single-PD sessions, + * matching Qualcomm's original implementation. */ + + /* Set client class */ + memset(&req, 0, sizeof(req)); + req.type = HAP_power_set_apptype; + req.apptype = HAP_POWER_COMPUTE_CLIENT_CLASS; + if (HAP_power_set((void *)&ctx->power_ctx, &req) != 0) { + GGMLHEXAGON_LOG_ERROR("HAP_power_set apptype failed"); + return -1; + } + + /* DCVS performance mode */ + memset(&req, 0, sizeof(req)); + req.type = HAP_power_set_DCVS_v3; + req.dcvs_v3.set_dcvs_enable = 1; + req.dcvs_v3.dcvs_enable = 0; // disable DVFS, pin to fixed frequency for stable performance + req.dcvs_v3.dcvs_option = HAP_DCVS_V2_PERFORMANCE_MODE; + req.dcvs_v3.set_bus_params = 1; + req.dcvs_v3.bus_params.min_corner = HAP_DCVS_VCORNER_MAX; + req.dcvs_v3.bus_params.max_corner = HAP_DCVS_VCORNER_MAX; + req.dcvs_v3.bus_params.target_corner = HAP_DCVS_VCORNER_MAX; + req.dcvs_v3.set_core_params = 1; + req.dcvs_v3.core_params.min_corner = HAP_DCVS_VCORNER_MAX; + req.dcvs_v3.core_params.max_corner = HAP_DCVS_VCORNER_MAX; + req.dcvs_v3.core_params.target_corner = HAP_DCVS_VCORNER_MAX; + req.dcvs_v3.set_sleep_disable = 1; + req.dcvs_v3.sleep_disable = 1; + + GGMLHEXAGON_LOG_INFO("__HVX_ARCH__ = %d\n", __HVX_ARCH__); + + // v79 architecture requires protected bus corners setting +#if __HEXAGON_ARCH__ >= 79 + HAP_set_dcvs_v3_protected_bus_corners(&req, 1); +#endif + + if (HAP_power_set((void *)&ctx->power_ctx, &req) != 0) { + GGMLHEXAGON_LOG_ERROR("HAP_power_set DCVS failed"); + return -2; + } + + /* Power up HVX */ + memset(&req, 0, sizeof(req)); + req.type = HAP_power_set_HVX; + req.hvx.power_up = 1; + if (HAP_power_set((void *)&ctx->power_ctx, &req) != 0) { + GGMLHEXAGON_LOG_ERROR("HAP_power_set HVX failed"); + return -3; + } + + /* Power up HMX with v2 settings for v75+ architecture */ +#if __HVX_ARCH__ >= 75 + memset(&req, 0, sizeof(req)); + req.type = HAP_power_set_HMX_v2; + req.hmx_v2.set_power = 1; + req.hmx_v2.power_up = 1; + req.hmx_v2.set_clock = 1; + req.hmx_v2.target_corner = HAP_DCVS_EXP_VCORNER_MAX; + req.hmx_v2.min_corner = HAP_DCVS_EXP_VCORNER_MAX; + req.hmx_v2.max_corner = HAP_DCVS_EXP_VCORNER_MAX; + req.hmx_v2.perf_mode = HAP_CLK_PERF_HIGH; + GGMLHEXAGON_LOG_INFO("Setting HMX clock with HMX_v2 for v75+ architecture"); + if (HAP_power_set((void *)&ctx->power_ctx, &req) != 0) { + GGMLHEXAGON_LOG_ERROR("HAP_power_set HMX_v2 failed, continuing without HMX"); + return -4; + } +#else + /* Power up HMX (legacy for older architectures) */ + memset(&req, 0, sizeof(req)); + req.type = HAP_power_set_HMX; + req.hmx.power_up = 1; + if (HAP_power_set((void *)&ctx->power_ctx, &req) != 0) { + GGMLHEXAGON_LOG_ERROR("HAP_power_set HMX failed, continuing without HMX"); + return -4; + } +#endif + + GGMLHEXAGON_LOG_INFO("HAP_power_set for HVX and HMX succeeded"); + return 0; +} + +static AEEResult hap_probe_dsp(remote_handle64 h) { + int retVal = 0; + + boolean dcvs_enabled; + unsigned int max_mips = 0; + unsigned int max_bus_bw = 0; + int client_class = 0; + unsigned int clk_freq_hz = 0; + void * context_ptr = NULL; + + HAP_power_response_t response; + context_ptr = g_dsp_ctx->hexagon_power_ctx; + + memset(&response, 0, sizeof(HAP_power_response_t)); + response.type = HAP_power_get_max_mips; + retVal = HAP_power_get(context_ptr, &response); + if (retVal!=AEE_SUCCESS) { + FARF(ERROR, "Unable to get the maximum MIPS supported"); + return AEE_EFAILED; + } + + max_mips = response.max_mips; + memset(&response, 0, sizeof(HAP_power_response_t)); + response.type = HAP_power_get_max_bus_bw; + retVal = HAP_power_get(context_ptr, &response); + if (retVal!=AEE_SUCCESS) { + FARF(ERROR, "Unable to get the maximum bus bandwidth supported"); + return AEE_EFAILED; + } + + max_bus_bw = response.max_bus_bw; + memset(&response, 0, sizeof(HAP_power_response_t)); + response.type = HAP_power_get_client_class; + retVal = HAP_power_get(context_ptr, &response); + if (retVal!=AEE_SUCCESS) { + FARF(ERROR, "Unable to get the client class"); + return AEE_EFAILED; + } + + client_class = response.client_class; + memset(&response, 0, sizeof(HAP_power_response_t)); + response.type = HAP_power_get_clk_Freq; + retVal = HAP_power_get(context_ptr, &response); + if (retVal!=AEE_SUCCESS) { + FARF(ERROR, "Unable to get the DSP core clock frequency"); + return AEE_EFAILED; + } + + clk_freq_hz = response.clkFreqHz; + memset(&response, 0, sizeof(HAP_power_response_t)); + response.type = HAP_power_get_dcvsEnabled; + retVal = HAP_power_get(context_ptr, &response); + if (retVal!=AEE_SUCCESS) { + FARF(ERROR, "Unable to get the DCVS status"); + return AEE_EFAILED; + } + + dcvs_enabled = response.dcvsEnabled; + printf("\nMaximum MIPS of DSP: %u" + "\nMaximum Bus Bandwidth supported: %u Bytes/second(%u MiB/s)" + "\nClient Class: %x" + "\nCore clock frequency of the DSP: %u" + "\nDCVS status: %d\n\n", + max_mips, max_bus_bw, max_bus_bw >> 20, client_class, clk_freq_hz, dcvs_enabled); + + return AEE_SUCCESS; +} + +// ================================================================================================= +// Qualcomm compatibility layer (ported from Qualcomm's ggml-hexagon) +// ================================================================================================= +// begin translation layer { + +// Adapters for the old htp_mm_hvx vtcm_sizes API that was replaced by +// htp_mm_hvx_vtcm_layout_build in upstream htp/matmul-ops.h. +static inline size_t htp_mm_hvx_get_vtcm_sizes( + int kernel_type, int wtype, uint32_t ne10, uint32_t src1_nrows, + uint32_t n_threads, + size_t dst_row_size, size_t src0_row_size, size_t src1_row_size, + uint32_t n_prefetch, + size_t * vtcm_src0_size, size_t * vtcm_src1_size, size_t * vtcm_dst_size +) { + struct htp_mm_hvx_vtcm_layout L; + htp_mm_hvx_vtcm_layout_build(&L, kernel_type, wtype, ne10, src1_nrows, n_threads, + dst_row_size, src0_row_size, src1_row_size, 0, n_prefetch, + false, false, false); + *vtcm_src0_size = L.src0_bytes; + *vtcm_src1_size = L.src1_bytes; + *vtcm_dst_size = L.dst_bytes; + return L.total_bytes; +} + +// =========================================================================== +// execute_op dispatch wrapper +// All op_xxx functions are exported from htp/*.c (non-static, declared in +// htp-ctx.h). We only need this dispatch wrapper + a translation layer. +// +// Function pointer table replaces the switch statement: eliminates branch +// prediction overhead (single indirect jump vs. chained conditional branches). +// Table is ~180 bytes, fits in L1I cache. Indexed directly by htp_op_code. +// =========================================================================== +typedef int (*htp_op_func_t)(struct htp_ops_context *); + +static const htp_op_func_t g_op_dispatch[HTP_OP_INVALID] = { + [HTP_OP_MUL] = op_binary, + [HTP_OP_ADD] = op_binary, + [HTP_OP_SUB] = op_binary, + [HTP_OP_DIV] = op_binary, + [HTP_OP_MUL_MAT] = op_matmul, + [HTP_OP_MUL_MAT_ID] = op_matmul_id, + [HTP_OP_MUL_MAT_QKV] = op_matmul_qkv, + [HTP_OP_MUL_MAT_FFN] = op_matmul_ffn, + [HTP_OP_MUL_MAT_ADD] = op_matmul, + [HTP_OP_NORM] = op_unary, + [HTP_OP_RMS_NORM] = op_unary, + [HTP_OP_RMS_NORM_MUL] = op_unary, + [HTP_OP_SCALE] = op_unary, + [HTP_OP_SQR] = op_unary, + [HTP_OP_SQRT] = op_unary, + [HTP_OP_UNARY_SOFTPLUS] = op_unary, + [HTP_OP_UNARY_SIGMOID] = op_unary, + [HTP_OP_UNARY_NEG] = op_unary, + [HTP_OP_UNARY_EXP] = op_unary, + [HTP_OP_UNARY_TANH] = op_unary, + [HTP_OP_L2_NORM] = op_unary, + [HTP_OP_UNARY_SILU] = op_unary, + [HTP_OP_UNARY_GELU] = op_unary, + [HTP_OP_GLU_SWIGLU] = op_activations, + [HTP_OP_GLU_SWIGLU_OAI] = op_activations, + [HTP_OP_GLU_GEGLU] = op_activations, + [HTP_OP_SOFTMAX] = op_softmax, + [HTP_OP_ADD_ID] = op_binary, + [HTP_OP_ROPE] = op_rope, + [HTP_OP_FLASH_ATTN_EXT] = op_flash_attn_ext, + [HTP_OP_SET_ROWS] = op_set_rows, + [HTP_OP_GET_ROWS] = op_get_rows, + [HTP_OP_SUM_ROWS] = op_sum_rows, + [HTP_OP_CPY] = op_cpy, + [HTP_OP_REPEAT] = op_repeat, + [HTP_OP_ARGSORT] = op_argsort, + [HTP_OP_SSM_CONV] = op_ssm_conv, + [HTP_OP_CUMSUM] = op_cumsum, + [HTP_OP_FILL] = op_fill, + [HTP_OP_DIAG] = op_diag, + [HTP_OP_SOLVE_TRI] = op_solve_tri, + [HTP_OP_PAD] = op_pad, + [HTP_OP_CONCAT] = op_concat, + [HTP_OP_IM2COL] = op_im2col, + [HTP_OP_GATED_DELTA_NET] = op_gated_delta_net, + [HTP_OP_TRI] = op_unary, +}; + +static int execute_op(struct htp_ops_context * octx) { +#if HEX_OP_PROF + const uint64_t t0 = ggml_time_us(); +#endif + const unsigned int op = (unsigned int) octx->op; + int ret; + if (op < HTP_OP_INVALID) { + htp_op_func_t fn = g_op_dispatch[op]; + if (fn) { + ret = fn(octx); + } else { + FARF(ERROR, "Unknown Op %u", op); + ret = -1; + } + } else { + FARF(ERROR, "Unknown Op %u", op); + ret = -1; + } +#if HEX_OP_PROF + { + const uint64_t dt = ggml_time_us() - t0; + if (op < HEX_OP_PROF_BUCKETS) { + g_dsp_ctx->op_prof_dur_us[op] += dt; + g_dsp_ctx->op_prof_count [op] += 1; + if (dt > g_dsp_ctx->op_prof_max_us[op]) g_dsp_ctx->op_prof_max_us[op] = dt; + if (dt < g_dsp_ctx->op_prof_min_us[op]) g_dsp_ctx->op_prof_min_us[op] = dt; + } + } +#endif + return ret; +} + +// Convert hex_tensor_desc to dsptensor in-place. Avoids the memset+memcpy +// pattern that was ~200 ns per tensor. The hex_tensor_desc layout is: +// {type, ne[4], nb[4], op_params[16], flags, data_offset, data_len} +// The dsptensor layout is: +// {type, ne[4], nb[4], op, op_params[16], flags, data, data_len} +// The only difference is dsptensor has an extra 'op' field between nb and +// op_params, and data is a pointer instead of an offset. +static inline void hex_tensor_to_dsptensor(const hex_tensor_desc * ht, + const char * ion_base, + dsptensor * dt) { + dt->type = ht->type; + dt->ne[0] = ht->ne[0]; dt->ne[1] = ht->ne[1]; + dt->ne[2] = ht->ne[2]; dt->ne[3] = ht->ne[3]; + dt->nb[0] = ht->nb[0]; dt->nb[1] = ht->nb[1]; + dt->nb[2] = ht->nb[2]; dt->nb[3] = ht->nb[3]; + dt->op = 0; // not used by execute_op path + memcpy(dt->op_params, ht->op_params, sizeof(dt->op_params)); + dt->flags = ht->flags; + dt->data = (void *)(ion_base + ht->data_offset); + dt->data_len = ht->data_len; +} + +// Convert hex_tensor_desc directly to htp_tensor. Eliminates the intermediate +// dsptensor step for op dispatch. Mirrors htp/main.c prep_tensor: data pointer +// is computed from mempool base + offset. flags is never read on the mempool path; +// cache coherency is handled by entry.c itself. +static inline void hex_tensor_to_htp_tensor(const hex_tensor_desc * ht, + const char * ion_base, + struct htp_tensor * htp) { + htp->data = (uint32_t)(uintptr_t)(ion_base + ht->data_offset); + htp->size = (uint32_t)ht->data_len; + htp->flags = HTP_TENSOR_FLUSHED; + htp->type = (uint16_t)ht->type; + htp->bi = 0; + htp->ne[0] = (uint32_t)ht->ne[0]; + htp->ne[1] = (uint32_t)ht->ne[1]; + htp->ne[2] = (uint32_t)ht->ne[2]; + htp->ne[3] = (uint32_t)ht->ne[3]; + htp->nb[0] = (uint32_t)ht->nb[0]; + htp->nb[1] = (uint32_t)ht->nb[1]; + htp->nb[2] = (uint32_t)ht->nb[2]; + htp->nb[3] = (uint32_t)ht->nb[3]; +} + +// Map GGML opcode to HTP opcode. Returns 0 on success, -1 if unsupported. +// For GGML_OP_UNARY, op_params[0] selects the unary sub-op. +static int ggml_op_to_htp_op(int32_t ggml_op, const int32_t * op_params, + enum htp_op_code * htp_op) { + switch (ggml_op) { + case GGML_OP_ADD: *htp_op = HTP_OP_ADD; return 0; + case GGML_OP_SUB: *htp_op = HTP_OP_SUB; return 0; + case GGML_OP_MUL: *htp_op = HTP_OP_MUL; return 0; + case GGML_OP_DIV: *htp_op = HTP_OP_DIV; return 0; + case GGML_OP_MUL_MAT: *htp_op = HTP_OP_MUL_MAT; return 0; + case GGML_OP_RMS_NORM: *htp_op = HTP_OP_RMS_NORM; return 0; + case GGML_OP_ROPE: *htp_op = HTP_OP_ROPE; return 0; + case GGML_OP_FLASH_ATTN_EXT: *htp_op = HTP_OP_FLASH_ATTN_EXT; return 0; + case GGML_OP_SOFT_MAX: *htp_op = HTP_OP_SOFTMAX; return 0; + case GGML_OP_SCALE: *htp_op = HTP_OP_SCALE; return 0; + case GGML_OP_CONCAT: *htp_op = HTP_OP_CONCAT; return 0; + case GGML_OP_CPY: *htp_op = HTP_OP_CPY; return 0; + case GGML_OP_GET_ROWS: *htp_op = HTP_OP_GET_ROWS; return 0; + case GGML_OP_SET_ROWS: *htp_op = HTP_OP_SET_ROWS; return 0; + case GGML_OP_SUM_ROWS: *htp_op = HTP_OP_SUM_ROWS; return 0; + case GGML_OP_SSM_CONV: *htp_op = HTP_OP_SSM_CONV; return 0; + case GGML_OP_CONT: *htp_op = HTP_OP_CPY; return 0; + case GGML_OP_REPEAT: *htp_op = HTP_OP_REPEAT; return 0; + case GGML_OP_NORM: *htp_op = HTP_OP_NORM; return 0; + case GGML_OP_L2_NORM: *htp_op = HTP_OP_L2_NORM; return 0; + case GGML_OP_SQR: *htp_op = HTP_OP_SQR; return 0; + case GGML_OP_SQRT: *htp_op = HTP_OP_SQRT; return 0; + case GGML_OP_ARGSORT: *htp_op = HTP_OP_ARGSORT; return 0; + case GGML_OP_PAD: *htp_op = HTP_OP_PAD; return 0; + case GGML_OP_IM2COL: *htp_op = HTP_OP_IM2COL; return 0; + case GGML_OP_GATED_DELTA_NET: *htp_op = HTP_OP_GATED_DELTA_NET; return 0; + case GGML_OP_CUMSUM: *htp_op = HTP_OP_CUMSUM; return 0; + case GGML_OP_FILL: *htp_op = HTP_OP_FILL; return 0; + case GGML_OP_DIAG: *htp_op = HTP_OP_DIAG; return 0; + case GGML_OP_TRI: *htp_op = HTP_OP_TRI; return 0; + case GGML_OP_SOLVE_TRI: *htp_op = HTP_OP_SOLVE_TRI; return 0; + case GGML_OP_UNARY: { + if (!op_params) { + FARF(ERROR, "ggml_op_to_htp_op: UNARY missing op_params"); + return -1; + } + switch (op_params[0]) { + case GGML_UNARY_OP_NEG: *htp_op = HTP_OP_UNARY_NEG; return 0; + case GGML_UNARY_OP_TANH: *htp_op = HTP_OP_UNARY_TANH; return 0; + case GGML_UNARY_OP_SIGMOID: *htp_op = HTP_OP_UNARY_SIGMOID; return 0; + case GGML_UNARY_OP_GELU: + case GGML_UNARY_OP_GELU_QUICK: *htp_op = HTP_OP_UNARY_GELU; return 0; + case GGML_UNARY_OP_SILU: *htp_op = HTP_OP_UNARY_SILU; return 0; + case GGML_UNARY_OP_EXP: *htp_op = HTP_OP_UNARY_EXP; return 0; + case GGML_UNARY_OP_SOFTPLUS: *htp_op = HTP_OP_UNARY_SOFTPLUS; return 0; + default: + FARF(ERROR, "ggml_op_to_htp_op: unsupported unary_op %d", op_params[0]); + return -1; + } + } + case GGML_OP_GLU: { + if (!op_params) { + FARF(ERROR, "ggml_op_to_htp_op: GLU missing op_params"); + return -1; + } + switch (op_params[0]) { + case GGML_GLU_OP_SWIGLU: *htp_op = HTP_OP_GLU_SWIGLU; return 0; + case GGML_GLU_OP_SWIGLU_OAI: *htp_op = HTP_OP_GLU_SWIGLU_OAI; return 0; + case GGML_GLU_OP_GEGLU: *htp_op = HTP_OP_GLU_GEGLU; return 0; + default: + FARF(ERROR, "ggml_op_to_htp_op: unsupported glu_op %d", op_params[0]); + return -1; + } + } + default: + FARF(ERROR, "ggml_op_to_htp_op: unsupported ggml_op %d", ggml_op); + return -1; + } +} + +// Build htp_ops_context directly from pre-converted g_dsp_ctx->pre_ht tensors. +// Eliminates per-op dsptensor_to_htp_tensor() calls and stack-allocated +// src_ht/dst_ht arrays. Mirrors htp/main.c proc_op_req: direct tensor +// table indexing (tens + op->src[i]). +static void build_htp_octx( + struct htp_ops_context * octx, + enum htp_op_code htp_op, + const int32_t * op_params, + const int32_t * kernel_params, + const int32_t src_idx[HTP_OP_MAX_INPUTS], + const int32_t dst_idx[HTP_OP_MAX_OUTPUTS]) { + + // Zero only spad pointers and flags; execute_op reads these + // unconditionally on some paths. Avoid full-struct memset which + // is wasted on fields we immediately overwrite. + octx->flags = 0; + octx->src0_spad.src = NULL; + octx->src1_spad.src = NULL; + octx->src2_spad.src = NULL; + octx->src3_spad.src = NULL; + octx->dst_spad.src = NULL; + + octx->ctx = g_dsp_ctx->htp_ctx; + octx->op = htp_op; + memcpy(octx->op_params, op_params, sizeof(octx->op_params)); + if (kernel_params) { + memcpy(octx->kernel_params, kernel_params, sizeof(octx->kernel_params)); + } else { + memset(octx->kernel_params, 0, sizeof(octx->kernel_params)); + } + + for (int i = 0; i < HTP_OP_MAX_INPUTS; i++) { + octx->src[i] = (src_idx[i] >= 0) ? &g_dsp_ctx->pre_ht[src_idx[i]] : NULL; + } + + for (int i = 0; i < HTP_OP_MAX_OUTPUTS; i++) { + octx->dsts[i] = (dst_idx[i] >= 0) ? &g_dsp_ctx->pre_ht[dst_idx[i]] : NULL; + } + + octx->n_threads = (uint32_t)g_dsp_ctx->thread_counts; +} + +// Try HMX precompute (simple 2D path). Mirrors ggml_hexagon_precompute_hmx_mm_params +// minus the grouped batched path: this DSP-side fallback handles plain MUL_MAT +// only; MUL_MAT_ID and fused matmuls rely on AP-precomputed kernel_params +// (kernel_type != 0) and are never rebuilt here. +// Returns true on success, false to fall back to HVX. +static bool build_mm_hmx_params(struct htp_ops_context * octx, + struct htp_mm_kernel_params * kparams) { + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * src1 = octx->src[1]; + + const int wtype = src0->type; + const uint32_t ne00 = src0->ne[0]; + const uint32_t ne01 = src0->ne[1]; + const uint32_t ne02 = src0->ne[2]; + const uint32_t ne03 = src0->ne[3]; + const uint32_t ne10 = src1->ne[0]; + const uint32_t ne11 = src1->ne[1]; + const uint32_t ne12 = src1->ne[2]; + const uint32_t ne13 = src1->ne[3]; + + const bool is_repack = (wtype == HTP_TYPE_Q4_0 || wtype == HTP_TYPE_Q4_1 || + wtype == HTP_TYPE_Q8_0 || wtype == HTP_TYPE_IQ4_NL || + wtype == HTP_TYPE_MXFP4); + const bool is_hmx_wtype = (wtype == HTP_TYPE_F16 || wtype == HTP_TYPE_F32 || is_repack); + if (!is_hmx_wtype) return false; + + const bool is_batched = (ne02 * ne03 > 1 || ne12 * ne13 > 1); + + const int ne00_padded = is_repack ? hex_round_up(ne00, 32) : (int) ne00; + const int ne01_padded = is_repack ? hex_round_up(ne01, 32) : (int) ne01; + const int ne11_padded = hex_round_up(ne11, 32); + + // Eligibility (mirrors ggml_hexagon_matmul_is_hmx_eligible) + if (ne01_padded % 32 != 0) return false; + if (ne00 % 32 != 0) return false; + if (is_batched && wtype != HTP_TYPE_F16) return false; + if (src0->nb[0] > src0->nb[1] || src1->nb[0] > src1->nb[1]) return false; + if (ne11 <= HTP_MM_HMX_MIN_NROWS) return false; + + const uint32_t aligned_tile_size = htp_mm_get_weight_aligned_tile_size(wtype); + const bool pipeline = htp_mm_hmx_pipeline(ne11); + const int n_threads = (int) octx->n_threads; + const size_t vtcm_budget = g_dsp_ctx->vtcm_size; + + size_t best_mblocks = SIZE_MAX; + int best_act_threads = 0; + size_t best_m_chunk = 0; + size_t best_n_chunk = 0; + size_t best_vtcm_size = 0; + + int act_threads = n_threads; + while (act_threads >= 1) { + const size_t act_f32_size = hex_align_up( + (size_t) act_threads * HTP_MM_DMA_ACT_MULTIPLIER * ne00_padded * sizeof(float), + HTP_MM_HMX_TILE_SIZE); + const size_t overhead = 256 + act_f32_size; + + size_t cost_n = 0, cost_m = 0, cost_mn = 0; + htp_mm_hmx_get_2d_chunk_costs(wtype, ne00_padded, pipeline, aligned_tile_size, + &cost_n, &cost_m, &cost_mn); + + size_t m_chunk_cand = 0, n_chunk_cand = 0, vtcm_size_cand = 0; + if (htp_mm_hmx_compute_chunks(vtcm_budget, overhead, cost_n, cost_m, cost_mn, + (size_t) ne11_padded, (size_t) ne01_padded, + (size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, + (size_t) ne11 * HTP_MM_HMX_COST_A_CONVERT, + &m_chunk_cand, &n_chunk_cand, &vtcm_size_cand) == 0) { + size_t exact_size = htp_mm_hmx_get_2d_vtcm_size( + wtype, ne00_padded, m_chunk_cand, n_chunk_cand, pipeline, + act_threads, aligned_tile_size); + if (exact_size <= vtcm_budget) { + size_t mblocks = ((size_t) ne11 + m_chunk_cand - 1) / m_chunk_cand; + if (mblocks < best_mblocks || + (mblocks == best_mblocks && act_threads > best_act_threads)) { + best_mblocks = mblocks; + best_act_threads = act_threads; + best_m_chunk = m_chunk_cand; + best_n_chunk = n_chunk_cand; + best_vtcm_size = exact_size; + } + } + } + if (act_threads == 1) break; + act_threads /= 2; + } + + if (best_act_threads == 0) return false; + + kparams->n_hmx = 1; + kparams->pipeline = pipeline ? 1 : 0; + kparams->m_chunk = (int32_t) best_m_chunk; + kparams->n_chunk = (int32_t) best_n_chunk; + kparams->n_threads = n_threads; + kparams->n_act_threads = best_act_threads; + kparams->tile_size = (int32_t) htp_mm_get_weight_tile_size(wtype); + kparams->aligned_tile_size = (int32_t) aligned_tile_size; + kparams->src1_row_size = (int32_t)((wtype == HTP_TYPE_Q4_1) + ? htp_mm_q8_1_tiled_row_size(ne10) + : htp_mm_q8_0_tiled_row_size(ne10)); + kparams->vtcm_size = (int32_t) best_vtcm_size; + kparams->vtcm_src0_size = 0; + kparams->vtcm_src1_size = 0; + kparams->vtcm_dst_size = 0; + kparams->n_prefetch = 16; + kparams->kernel_type = is_batched ? HTP_MM_KERNEL_HMX_F16_BATCHED + : HTP_MM_KERNEL_HMX_2D; + // Used by HMX kernels for activation DMA/work split (matmul-ops.c). + kparams->div_n_act_threads = init_fastdiv_values((uint32_t) best_act_threads); + kparams->div_ne00_padded = init_fastdiv_values((uint32_t) ne00_padded); + return true; +} + +// Compute htp_mm_kernel_params on DSP side for MUL_MAT. +// Tries HMX first (if available), falls back to HVX F32/F16/quantized paths. +static int build_mm_kernel_params(struct htp_ops_context * octx) { + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * src1 = octx->src[1]; + const struct htp_tensor * dst = octx->dst; + if (!src0 || !src1 || !dst) return -1; + + struct htp_mm_kernel_params * kparams = + (struct htp_mm_kernel_params *) octx->kernel_params; + + // If AP side already precomputed kernel params (kernel_type != 0), + // skip DSP-side recomputation. The AP side uses the same HMX-first + // then HVX-fallback policy (ggml_hexagon_precompute_mm_params) + // as this function, and may have precomputed HMX chunk sizes that + // the DSP-side build_mm_hmx_params would otherwise recompute. + if (kparams->kernel_type != 0) { + return 0; + } + + memset(kparams, 0, sizeof(*kparams)); + + const int wtype = src0->type; + const uint32_t ne02 = src0->ne[2]; + const uint32_t ne03 = src0->ne[3]; + const uint32_t ne10 = src1->ne[0]; + const uint32_t ne11 = src1->ne[1]; + const uint32_t ne12 = src1->ne[2]; + const uint32_t ne13 = src1->ne[3]; + const uint32_t src1_nrows = ne11 * ne12 * ne13; + + kparams->n_hmx = 0; + kparams->n_threads = octx->n_threads; + kparams->n_prefetch = 16; + + // Try HMX first (mirrors ggml_hexagon_precompute_mm_params: HMX-first, HVX-fallback) + if (g_dsp_ctx->hmx_available && build_mm_hmx_params(octx, kparams)) { + goto mm_finalize; + } + + const bool is_batched = (ne02 > 1) || (ne03 > 1); + const bool is_permuted = (src0->nb[0] > src0->nb[1] || src0->nb[1] > src0->nb[2] || src0->nb[2] > src0->nb[3]) || + (src1->nb[0] > src1->nb[1] || src1->nb[1] > src1->nb[2] || src1->nb[2] > src1->nb[3]); + + size_t vtcm_src0_size = 0, vtcm_src1_size = 0, vtcm_dst_size = 0; + + if (wtype == HTP_TYPE_F32) { + size_t vtcm_size = htp_mm_hvx_get_vtcm_sizes( + HTP_MM_KERNEL_HVX_F32_F32_VTCM, wtype, ne10, src1_nrows, octx->n_threads, + dst->nb[1], src0->nb[1], src1->nb[1], 16, + &vtcm_src0_size, &vtcm_src1_size, &vtcm_dst_size); + + if (!is_batched && !is_permuted && vtcm_size <= g_dsp_ctx->vtcm_size) { + kparams->kernel_type = HTP_MM_KERNEL_HVX_F32_F32_VTCM; + kparams->src1_row_size = hex_round_up(ne10 * 4, 128); + } else { + kparams->kernel_type = HTP_MM_KERNEL_HVX_F32_F32_DDR; + kparams->src1_row_size = src1->nb[1]; + vtcm_size = htp_mm_hvx_get_vtcm_sizes( + kparams->kernel_type, wtype, ne10, src1_nrows, octx->n_threads, + dst->nb[1], src0->nb[1], src1->nb[1], 16, + &vtcm_src0_size, &vtcm_src1_size, &vtcm_dst_size); + } + kparams->vtcm_size = (int32_t) vtcm_size; + kparams->vtcm_src0_size = (int32_t) vtcm_src0_size; + kparams->vtcm_src1_size = (int32_t) vtcm_src1_size; + kparams->vtcm_dst_size = (int32_t) vtcm_dst_size; + } else if (wtype == HTP_TYPE_F16) { + size_t vtcm_size = htp_mm_hvx_get_vtcm_sizes( + HTP_MM_KERNEL_HVX_F16_F16_VTCM, wtype, ne10, src1_nrows, octx->n_threads, + dst->nb[1], src0->nb[1], src1->nb[1], 16, + &vtcm_src0_size, &vtcm_src1_size, &vtcm_dst_size); + + if (!is_batched && !is_permuted && vtcm_size <= g_dsp_ctx->vtcm_size) { + kparams->kernel_type = HTP_MM_KERNEL_HVX_F16_F16_VTCM; + kparams->src1_row_size = hex_round_up(ne10 * 2, 128); + } else { + if (src1->type == HTP_TYPE_F32) { + kparams->kernel_type = HTP_MM_KERNEL_HVX_F16_F32_DDR; + } else { + kparams->kernel_type = HTP_MM_KERNEL_HVX_F16_F16_DDR; + } + kparams->src1_row_size = src1->nb[1]; + vtcm_size = htp_mm_hvx_get_vtcm_sizes( + kparams->kernel_type, wtype, ne10, src1_nrows, octx->n_threads, + dst->nb[1], src0->nb[1], src1->nb[1], 16, + &vtcm_src0_size, &vtcm_src1_size, &vtcm_dst_size); + } + kparams->vtcm_size = (int32_t) vtcm_size; + kparams->vtcm_src0_size = (int32_t) vtcm_src0_size; + kparams->vtcm_src1_size = (int32_t) vtcm_src1_size; + kparams->vtcm_dst_size = (int32_t) vtcm_dst_size; + } else { + // Quantized HVX path (Q4_0, Q4_1, Q5_0, Q8_0, IQ4_NL, MXFP4) + kparams->tile_size = (int32_t) htp_mm_get_weight_tile_size(wtype); + kparams->aligned_tile_size = (int32_t) htp_mm_get_weight_aligned_tile_size(wtype); + + const bool k_align = (ne10 % 32 == 0); + const bool try_tiled = k_align && kparams->tile_size > 0; + bool tiled_ok = false; + + if (try_tiled) { + kparams->src1_row_size = (int32_t)((wtype == HTP_TYPE_Q4_1) + ? htp_mm_q8_1_tiled_row_size(ne10) + : htp_mm_q8_0_tiled_row_size(ne10)); + kparams->kernel_type = (src1_nrows < octx->n_threads) + ? HTP_MM_KERNEL_HVX_QUANT_BLOCK + : HTP_MM_KERNEL_HVX_QUANT_ROW; + + const uint32_t max_prefetch = (src1_nrows > HTP_MM_HMX_MIN_NROWS) ? 2 : 16; + uint32_t best_n_prefetch = 2; + size_t vs0 = 0, vs1 = 0, vd = 0; + size_t total_size = 0; + for (uint32_t d = max_prefetch; d >= 2; d /= 2) { + total_size = htp_mm_hvx_get_vtcm_sizes( + kparams->kernel_type, wtype, ne10, src1_nrows, octx->n_threads, + dst->nb[1], src0->nb[1], src1->nb[1], d, + &vs0, &vs1, &vd); + if (total_size <= g_dsp_ctx->vtcm_size) { + best_n_prefetch = d; + break; + } + } + if (best_n_prefetch == 2 && total_size > g_dsp_ctx->vtcm_size) { + total_size = htp_mm_hvx_get_vtcm_sizes( + kparams->kernel_type, wtype, ne10, src1_nrows, octx->n_threads, + dst->nb[1], src0->nb[1], src1->nb[1], 2, + &vs0, &vs1, &vd); + } + kparams->n_prefetch = (int32_t) best_n_prefetch; + + if (total_size <= g_dsp_ctx->vtcm_size) { + kparams->vtcm_size = (int32_t) total_size; + kparams->vtcm_src0_size = (int32_t) vs0; + kparams->vtcm_src1_size = (int32_t) vs1; + kparams->vtcm_dst_size = (int32_t) vd; + tiled_ok = true; + } + } + + if (!tiled_ok) { + kparams->src1_row_size = (int32_t)((wtype == HTP_TYPE_Q4_1) + ? htp_mm_q8_1_flat_row_size(ne10) + : htp_mm_q8_0_flat_row_size(ne10)); + kparams->kernel_type = HTP_MM_KERNEL_HVX_QUANT_ROW_FLAT; + + size_t vs0 = 0, vs1 = 0, vd = 0; + const size_t total_size = htp_mm_hvx_get_vtcm_sizes( + kparams->kernel_type, wtype, ne10, src1_nrows, octx->n_threads, + dst->nb[1], src0->nb[1], src1->nb[1], 16, + &vs0, &vs1, &vd); + + kparams->n_prefetch = 16; + kparams->vtcm_size = (int32_t) total_size; + kparams->vtcm_src0_size = (int32_t) vs0; + kparams->vtcm_src1_size = (int32_t) vs1; + kparams->vtcm_dst_size = (int32_t) vd; + } + } + +mm_finalize: + kparams->div_ne12_ne1 = init_fastdiv_values(ne12 * ne11); + kparams->div_ne1 = init_fastdiv_values(ne11); + kparams->div_r2 = init_fastdiv_values(ne02 > 0 ? ne12 / ne02 : 1); + kparams->div_r3 = init_fastdiv_values(ne03 > 0 ? ne13 / ne03 : 1); + kparams->div_ne11 = init_fastdiv_values(ne11); + + return 0; +} + +// end translation layer } + + +// ================================================================================================= +// IDL implementation +// ================================================================================================= +int ggml_dsp_open(const char * uri, remote_handle64 * handle) { + struct dsp_context * ctx = NULL; + + // Guard against double initialization + if (g_dsp_ctx != NULL) { + GGMLHEXAGON_LOG_ERROR("ggml_dsp_open: g_dsp_ctx already initialized"); + return AEE_EITEMBUSY; + } + + ctx = (struct dsp_context *)calloc(1, sizeof(struct dsp_context)); + GGML_ASSERT(NULL != ctx); + ctx->thread_counts = 4; + ctx->htp_ctx = (struct htp_context *)calloc(1, sizeof(struct htp_context)); + GGML_ASSERT(NULL != ctx->htp_ctx); + + // Allocate large per-session arrays from a single mempool. + // pre_dt and pre_ht are [DSP_OPT_MAX_TENSORS]; prior_dst_ranges and + // bulk_flush_ranges are [DSP_OPT_MAX_BATCH_DSTS]. + { + size_t pre_dt_sz = DSP_OPT_MAX_TENSORS * sizeof(dsptensor); + size_t pre_ht_sz = DSP_OPT_MAX_TENSORS * sizeof(struct htp_tensor); + size_t prior_sz = DSP_OPT_MAX_BATCH_DSTS * sizeof(dsp_dst_range_t); + size_t bulk_sz = DSP_OPT_MAX_BATCH_DSTS * sizeof(dsp_dst_range_t); + // align each segment to 128 bytes + size_t a_pre_dt = (pre_dt_sz + 127) & ~(size_t)127; + size_t a_pre_ht = (pre_ht_sz + 127) & ~(size_t)127; + size_t a_prior = (prior_sz + 127) & ~(size_t)127; + size_t a_bulk = (bulk_sz + 127) & ~(size_t)127; + size_t pool_sz = a_pre_dt + a_pre_ht + a_prior + a_bulk; + + ctx->arrays_pool = memalign(128, pool_sz); + if (!ctx->arrays_pool) { + GGMLHEXAGON_LOG_ERROR("arrays_pool memalign failed\n"); + free(ctx->htp_ctx); + free(ctx); + *handle = 0; + return AEE_EFAILED; + } + memset(ctx->arrays_pool, 0, pool_sz); + + char * p = (char *)ctx->arrays_pool; + ctx->pre_dt = (dsptensor *)p; p += a_pre_dt; + ctx->pre_ht = (struct htp_tensor *)p; p += a_pre_ht; + ctx->prior_dst_ranges = (dsp_dst_range_t *)p; p += a_prior; + ctx->bulk_flush_ranges = (dsp_dst_range_t *)p; + } + + ctx->thread_prio = qurt_thread_get_priority(qurt_thread_get_id()); + *handle = (remote_handle64)ctx; + + // Reset the first-touch weight tracking so a fresh session starts with + // no weights marked. Tracking is session-scoped: AP repacks/writes + // weights into the mempool at model load and their offsets are stable + // for the lifetime of this session. + // Use ctx directly: g_dsp_ctx is not set until after init completes. + ctx->weight_inval_count = 0; +#if HEX_OP_PROF + // Use ctx directly: g_dsp_ctx is not set until after init completes. + for (unsigned int i = 0; i < HEX_OP_PROF_BUCKETS; i++) { + ctx->op_prof_min_us[i] = UINT64_MAX; + } +#endif + // Default to 0: no DSP-side cache optimizations beyond the baseline + // first-touch weight tracking. AP will push the configured bitmask via + // execute_batch(0xFFFC) right after ggml_dsp_open returns. Until then, + // every code path that consults dsp_cache_mode sees 0 and behaves like + // baseline 29c1cf196. + ctx->dsp_cache_mode = 0; + ctx->dsp_cache_trace_bit0 = 0; // default off; AP pushes via 0xFFFC bit 16 + ctx->dsp_cache_trace_bit1 = 0; // default off; AP pushes via 0xFFFC bit 17 + + printf("uri %s\n", uri); + GGMLHEXAGON_LOG_INFO("uri = %s\n", uri); + GGMLHEXAGON_LOG_ALWAYS("uri = %s\n", uri); + + unsigned int api_version = qurt_api_version(); + printf("qurt_api_version = 0x%x\n", api_version); + printf("qurt_hvx_units = 0x%d\n", qurt_hvx_get_units()); + qurt_arch_version_t vers; + qurt_sysenv_get_arch_version(&vers); + printf("qurt_arch_version = 0x%x\n", vers.arch_version); + qurt_sysenv_app_heap_t aheap; + qurt_sysenv_get_app_heap(&aheap); + printf("aheap.heap_base=0x%x, aheap.heap_limit=0x%x\n", aheap.heap_base, aheap.heap_limit); + qurt_sysenv_max_hthreads_t mhwt; + qurt_sysenv_get_max_hw_threads(&mhwt); + printf("qurt_hardware_thread_counts = %d\n", mhwt.max_hthreads); + ctx->max_hw_threads = mhwt.max_hthreads; + ctx->thread_counts = mhwt.max_hthreads; + uint32_t hw_nhvx = (qurt_hvx_get_units() >> 8) & 0xFF; + printf("hw_nhvx = %lu\n", hw_nhvx); + + /* Step 1: Power up HVX and HMX */ + int power_result = power_on_hvx_hmx(ctx); + if (power_result != 0) { + printf("power_on_hvx_hmx failed (%d), continuing without HMX\n", power_result); + ctx->hmx_available = 0; + } else { + ctx->hmx_available = 1; + } + + /* Step 2: Query VTCM size and allocate resources */ + unsigned int vtcm_size_query = 0; + unsigned int availBlockSize; + compute_res_vtcm_page_t availBlock; + compute_res_vtcm_page_t totalBlock; + int result = 0; + result = HAP_compute_res_query_VTCM(0, &vtcm_size_query, &totalBlock, &availBlockSize, &availBlock); + GGMLHEXAGON_LOG_INFO("VTCM total = %u bytes\n", vtcm_size_query); + printf("Querying VTCM before acquiring resources:\n"); + printf("Compute resource query return %d, vtcm_size_query %d, availBlockSize %d\n", + result, vtcm_size_query, availBlockSize); + printf("Compute resource query ctd, valid page sizes in total table: %d, valid page sizes in avail table: %d\n", + totalBlock.page_list_len, availBlock.page_list_len); + if (totalBlock.page_list_len >= 1 && availBlock.page_list_len >= 2) { + printf("Compute resource query ctd, (Size, num pages); total (0x%x, %d) Avail (0x%x, %d, 0x%x, %d)\n", + totalBlock.page_list[0].page_size, + totalBlock.page_list[0].num_pages, + availBlock.page_list[0].page_size, + availBlock.page_list[0].num_pages, + availBlock.page_list[1].page_size, + availBlock.page_list[1].num_pages); + } else if (totalBlock.page_list_len >= 1 && availBlock.page_list_len >= 1) { + printf("Compute resource query ctd, (Size, num pages); total (0x%x, %d) Avail (0x%x, %d)\n", + totalBlock.page_list[0].page_size, + totalBlock.page_list[0].num_pages, + availBlock.page_list[0].page_size, + availBlock.page_list[0].num_pages); + } else { + printf("Compute resource query ctd, no page list data available\n"); + } + + /* Step 3: Acquire compute resources (including VTCM and HMX) */ + compute_res_attr_t attr; + unsigned int vtcm_size_to_use = (DEFAULT_VTCM_SIZE < vtcm_size_query) ? DEFAULT_VTCM_SIZE : vtcm_size_query; + HAP_compute_res_attr_init(&attr); + HAP_compute_res_attr_set_serialize(&attr, 0); + HAP_compute_res_attr_set_cache_mode(&attr, 1); // Enable cache mode (matching official implementation) + HAP_compute_res_attr_set_vtcm_param_v2(&attr, vtcm_size_to_use, vtcm_size_to_use, vtcm_size_to_use); // single page (matching official implementation) + HAP_compute_res_attr_set_release_callback(&attr, vtcm_release_callback, NULL); // Enable release callback for cache mode + HAP_compute_res_attr_set_hmx_param(&attr, 1); + // Allocate VTCM for scratch pads + ctx->compute_res_ctx_id = HAP_compute_res_acquire(&attr, 1000000); + if (ctx->compute_res_ctx_id == 0) { + GGMLHEXAGON_LOG_ERROR("HAP_compute_res_acquire failed, no VTCM available\n"); + free(ctx->arrays_pool); + free(ctx->htp_ctx); + free(ctx); + *handle = 0; + return AEE_EFAILED; + } else { + /* Using VTCM acquired via HAP_compute_res */ + void * vtcm_ptr = NULL; + unsigned int vtcm_ptr_size = 0; + if (HAP_compute_res_attr_get_vtcm_ptr_v2(&attr, &vtcm_ptr, &vtcm_ptr_size) != 0) { + GGMLHEXAGON_LOG_ERROR("HAP_compute_res_attr_get_vtcm_ptr_v2 failed\n"); + HAP_compute_res_release(ctx->compute_res_ctx_id); + free(ctx->arrays_pool); + free(ctx->htp_ctx); + free(ctx); + *handle = 0; + return AEE_EFAILED; + } else { + ctx->vtcm_base = vtcm_ptr; + ctx->vtcm_size = vtcm_ptr_size; + GGMLHEXAGON_LOG_INFO("allocated VTCM pool via compute_res: %zu bytes at %p\n", ctx->vtcm_size, ctx->vtcm_base); + + /* VTCM: acquire once for the whole session (matches + * htp/main.c htp_packet_callback pattern: acquire once at + * session start, release once at session end). + * Avoids per-batch HAP_compute_res_acquire/release_cached churn + * (~8700 calls/session -> 2 calls/session) and the SDK's + * adsprpc FARF log noise. */ + /* dsp_vtcm_acquire() uses g_dsp_ctx internally: publish ctx + * now that all fields it reads are initialized. */ + g_dsp_ctx = ctx; + dsp_vtcm_acquire(); + } + } + + /* Step 4: Create async HMX queue for pipeline overlap (DMA/HVX/HMX). + * New Qualcomm API (b2dd28a3b) requires a pre-allocated backing buffer; + * we memalign it here and track it in dsp_context.hmx_queue_buf for + * cleanup in ggml_dsp_close. Capacity/stack_size match main.c defaults. */ + if (ctx->hmx_available && ctx->compute_res_ctx_id != 0) { + if (ctx->hmx_queue != NULL) { + GGMLHEXAGON_LOG_INFO("hmx_queue already exists, deleting old one\n"); + hmx_queue_free(ctx->hmx_queue); + free(ctx->hmx_queue_buf); + ctx->hmx_queue = NULL; + ctx->hmx_queue_buf = NULL; + } + size_t hmx_size = hmx_queue_sizeof(HMX_QUEUE_CAPACITY, HMX_QUEUE_STACK_SIZE); + size_t hmx_align = hmx_queue_alignof(); + void * hmx_buf = memalign(hmx_align, hmx_size); + if (hmx_buf) { + // Trace slot mirrors htp/main.c (&ctx->trace[HTP_MAX_NTHREADS]): must be a + // valid (zeroed) htp_thread_trace, htp_trace_event_start/stop dereference + // it unconditionally on every HMX descriptor completion. + ctx->hmx_queue = hmx_queue_init(hmx_buf, HMX_QUEUE_CAPACITY, + HMX_QUEUE_STACK_SIZE, + ctx->compute_res_ctx_id, + &ctx->htp_ctx->trace[HTP_MAX_NTHREADS]); + if (ctx->hmx_queue) { + ctx->hmx_queue_buf = hmx_buf; + GGMLHEXAGON_LOG_INFO("async HMX queue created (capacity %u, rctx %u)\n", + hmx_queue_capacity(ctx->hmx_queue), ctx->compute_res_ctx_id); + } else { + free(hmx_buf); + ctx->hmx_queue_buf = NULL; + GGMLHEXAGON_LOG_INFO("hmx_queue_init failed, HMX path will run synchronously\n"); + } + } else { + GGMLHEXAGON_LOG_INFO("memalign for hmx_queue failed, HMX path will run synchronously\n"); + } + } else { + GGMLHEXAGON_LOG_INFO("HMX not available (hmx=%d, rctx=%u), skipping hmx_queue creation\n", + ctx->hmx_available, ctx->compute_res_ctx_id); + } + + /* Step 5: probe DSP memory for information only (no allocation) */ + { + struct HAP_mem_stats mem_stats; + memset(&mem_stats, 0, sizeof(mem_stats)); + int ret = HAP_mem_get_stats(&mem_stats); + if (ret == 0) { + printf("DSP HAP_mem_stats: bytes_free=%llu, bytes_used=%llu, seg_free=%llu, seg_used=%llu\n", + (unsigned long long)mem_stats.bytes_free, (unsigned long long)mem_stats.bytes_used, + (unsigned long long)mem_stats.seg_free, (unsigned long long)mem_stats.seg_used); + } else { + printf("HAP_mem_get_stats failed: %d\n", ret); + } + + // Probe available DSP heap (information only, no allocation) + size_t max_avail_mb = 0; + for (int mb = 2048; mb >= 16; mb -= 16) { + void * ptr = malloc((size_t)mb * 1024 * 1024); + if (ptr) { + printf("DSP malloc probe: %d MB succeeded at %p\n", mb, ptr); + free(ptr); + max_avail_mb = mb; + break; + } + } + if (max_avail_mb == 0) { + printf("DSP malloc probe: even 16 MB failed!\n"); + } else { + printf("DSP malloc probe: max available = %zu MB (for work data only, cache uses ION)\n", max_avail_mb); + } + } + + /* Publish ctx to g_dsp_ctx for VTCM-failure path. In the success path, + * g_dsp_ctx was already set before dsp_vtcm_acquire(); this assignment + * is a no-op there. */ + g_dsp_ctx = ctx; + return 0; +} + +int ggml_dsp_close(remote_handle64 handle) { + struct dsp_context * ctx = (struct dsp_context *)handle; + if (!ctx) return 0; + g_dsp_ctx = ctx; + + // Cleanup htp_context resources (work_queue + dma queues). + // New Qualcomm API (b2dd28a3b): *_queue_free() does not free the backing + // buffer, so we track them in dsp_context and free them explicitly here. + if (ctx->arrays_pool) { + free(ctx->arrays_pool); + ctx->arrays_pool = NULL; + } + if (ctx->htp_ctx) { + if (ctx->htp_ctx->work_queue) { + work_queue_free(ctx->htp_ctx->work_queue); + ctx->htp_ctx->work_queue = NULL; + } + if (ctx->work_queue_buf) { + free(ctx->work_queue_buf); + ctx->work_queue_buf = NULL; + } + for (int i = 0; i < HTP_MAX_NTHREADS; i++) { + if (ctx->htp_ctx->dma[i]) { + dma_queue_alias_free(ctx->htp_ctx->dma[i]); + ctx->htp_ctx->dma[i] = NULL; + } + if (ctx->dma_alias_bufs[i]) { + free(ctx->dma_alias_bufs[i]); + ctx->dma_alias_bufs[i] = NULL; + } + if (ctx->htp_ctx->dma_cached[i]) { + dma_queue_free(ctx->htp_ctx->dma_cached[i]); + ctx->htp_ctx->dma_cached[i] = NULL; + } + if (ctx->dma_queue_bufs[i]) { + free(ctx->dma_queue_bufs[i]); + ctx->dma_queue_bufs[i] = NULL; + } + } + free(ctx->htp_ctx); + ctx->htp_ctx = NULL; + } + + if (ctx->hmx_queue != NULL) { + hmx_queue_free(ctx->hmx_queue); + ctx->hmx_queue = NULL; + if (ctx->hmx_queue_buf) { + free(ctx->hmx_queue_buf); + ctx->hmx_queue_buf = NULL; + } + GGMLHEXAGON_LOG_INFO("released async HMX queue"); + } + + /* VTCM: release once at session end (matches the pattern, see + * ggml_dsp_open). The release callback is still registered and will + * set vtcm_needs_release=1 if another session preempts during the + * session, but we intentionally ignore it here so that VTCM stays + * cached for the full session. */ + dsp_vtcm_release(); + + if (ctx->compute_res_ctx_id != 0) { + // HAP_compute_res_release_cached is already called inside dsp_vtcm_release() + // NOTE: HMX lock is managed per-operation in mulmat.c, not here + // HAP_compute_res_hmx_unlock(ctx->compute_res_ctx_id); + + HAP_compute_res_release(ctx->compute_res_ctx_id); + ctx->compute_res_ctx_id = 0; + ctx->vtcm_base = NULL; + ctx->vtcm_size = 0; + GGMLHEXAGON_LOG_INFO("released compute resources"); + } + + g_dsp_ctx = NULL; + free(ctx); + return 0; +} + +AEEResult ggml_dsp_setclocks(remote_handle64 handle, int32 diag_info, int32 requested_thread_counts, int32 * actual_thread_counts) { + g_dsp_ctx = (struct dsp_context *)handle; + if (!g_dsp_ctx) return AEE_EBADPARM; + /* Reserve 2 hw thread slots: one for the hmx_queue thread, one for + * FastRPC listener/system activity. An op needs requested_thread_counts+1 + * co-resident threads; oversubscribing deadlocks because QuRT does + * not preempt equal-priority workers spinning in hex_pause, so the + * unscheduled worker never decrements the task barrier (observed on + * v75/8Gen3: max_hthreads=6, requested_thread_counts=6 hangs; 5 is flaky). */ + int max_usable = (g_dsp_ctx->max_hw_threads > 2) ? (g_dsp_ctx->max_hw_threads - 2) : 0; + if (max_usable > HTP_MAX_NTHREADS) max_usable = HTP_MAX_NTHREADS; + if (requested_thread_counts > max_usable) { + printf("setclocks: requested_thread_counts %ld exceeds safe limit %d (max_hthreads %d - 2), clamped\n", + requested_thread_counts, max_usable, g_dsp_ctx->max_hw_threads); + requested_thread_counts = max_usable; + } + g_dsp_ctx->thread_counts = requested_thread_counts; + g_dsp_ctx->dump_diag_info = diag_info; + + // Expose the actual thread count in effect on DSP side so AP can mirror it + // (avoids jobs > work-queue threads when n_act_threads is precomputed on AP). + if (actual_thread_counts) { + *actual_thread_counts = (int32_t)g_dsp_ctx->thread_counts; + } + + printf("\n"); + printf("actual thread_counts: %d\n", g_dsp_ctx->thread_counts); + printf("dump_diag_info: %d\n\n", g_dsp_ctx->dump_diag_info); + + // Initialize htp_context for calling the shared execute_op. + // Shares our already-acquired VTCM and HMX queue. + // New Qualcomm API (b2dd28a3b) requires pre-allocated backing buffers for + // work_queue and dma queues; we memalign them here and track the pointers + // in dsp_context for cleanup in ggml_dsp_close. + if (g_dsp_ctx->thread_counts >= 1) { + // Free previously allocated htp_ctx resources before memset, + // in case ggml_dsp_setclocks is called more than once. + if (g_dsp_ctx->htp_ctx->work_queue) { + work_queue_free(g_dsp_ctx->htp_ctx->work_queue); + g_dsp_ctx->htp_ctx->work_queue = NULL; + } + if (g_dsp_ctx->work_queue_buf) { + free(g_dsp_ctx->work_queue_buf); + g_dsp_ctx->work_queue_buf = NULL; + } + for (int i = 0; i < HTP_MAX_NTHREADS; i++) { + if (g_dsp_ctx->htp_ctx->dma[i]) { + dma_queue_alias_free(g_dsp_ctx->htp_ctx->dma[i]); + g_dsp_ctx->htp_ctx->dma[i] = NULL; + } + if (g_dsp_ctx->dma_alias_bufs[i]) { + free(g_dsp_ctx->dma_alias_bufs[i]); + g_dsp_ctx->dma_alias_bufs[i] = NULL; + } + if (g_dsp_ctx->htp_ctx->dma_cached[i]) { + dma_queue_free(g_dsp_ctx->htp_ctx->dma_cached[i]); + g_dsp_ctx->htp_ctx->dma_cached[i] = NULL; + } + if (g_dsp_ctx->dma_queue_bufs[i]) { + free(g_dsp_ctx->dma_queue_bufs[i]); + g_dsp_ctx->dma_queue_bufs[i] = NULL; + } + } + memset(g_dsp_ctx->htp_ctx, 0, sizeof(*g_dsp_ctx->htp_ctx)); + g_dsp_ctx->htp_ctx->vtcm_base = (uint8_t *)g_dsp_ctx->vtcm_base; + g_dsp_ctx->htp_ctx->vtcm_size = g_dsp_ctx->vtcm_size; + g_dsp_ctx->htp_ctx->vtcm_rctx = g_dsp_ctx->compute_res_ctx_id; + g_dsp_ctx->htp_ctx->hmx_queue = g_dsp_ctx->hmx_queue; + g_dsp_ctx->htp_ctx->n_threads = (uint32_t)g_dsp_ctx->thread_counts; + g_dsp_ctx->htp_ctx->n_threads_div = init_fastdiv_values((uint32_t)g_dsp_ctx->thread_counts); + g_dsp_ctx->htp_ctx->hmx_enabled = g_dsp_ctx->hmx_available ? true : false; + + // work_queue: backing buffer holds worker stacks + queue struct. + size_t wq_size = work_queue_sizeof((uint32_t)g_dsp_ctx->thread_counts, + WORK_QUEUE_CAPACITY, WORK_QUEUE_STACK_SIZE); + size_t wq_align = work_queue_alignof(); + void * wq_buf = memalign(wq_align, wq_size); + AEEResult wp = AEE_SUCCESS; + if (wq_buf) { + g_dsp_ctx->htp_ctx->work_queue = work_queue_init(wq_buf, + (uint32_t)g_dsp_ctx->thread_counts, + WORK_QUEUE_CAPACITY, + WORK_QUEUE_STACK_SIZE); + if (g_dsp_ctx->htp_ctx->work_queue) { + g_dsp_ctx->work_queue_buf = wq_buf; + } else { + free(wq_buf); + g_dsp_ctx->work_queue_buf = NULL; + GGMLHEXAGON_LOG_ERROR("work_queue_init failed"); + return AEE_EFAILED; + } + } else { + g_dsp_ctx->htp_ctx->work_queue = NULL; + g_dsp_ctx->work_queue_buf = NULL; + GGMLHEXAGON_LOG_ERROR("work_queue memalign failed"); + return AEE_ENOMEMORY; + } + printf("htp_ctx work_queue_init returned %d (n_threads=%d)\n", wp, g_dsp_ctx->thread_counts); + + // dma queues: one main queue (dma_cached) + one nocache alias (dma) per + // thread, mirroring main.c. Ops use the alias with nocache=1 so DMA DDR + // accesses bypass L2 (same semantics as the pre-b2dd28a3b hex-dma, which + // hardcoded bypass=1) and stay coherent with our manual dcinva/dccleaninva + // cache management. dma_queue_init must get a valid (zeroed) trace: + // htp_trace_event_start/stop dereference it unconditionally on every + // push/pop. + size_t dma_size = dma_queue_sizeof(256); + size_t dma_alias_size = dma_queue_alias_sizeof(); + size_t dma_align = dma_queue_alignof(); + for (int i = 0; i < g_dsp_ctx->thread_counts; i++) { + void * dma_buf = memalign(dma_align, dma_size); + if (dma_buf) { + g_dsp_ctx->htp_ctx->dma_cached[i] = dma_queue_init(dma_buf, 256, + (uintptr_t)g_dsp_ctx->vtcm_base, + g_dsp_ctx->vtcm_size, + &g_dsp_ctx->htp_ctx->trace[i]); + if (g_dsp_ctx->htp_ctx->dma_cached[i]) { + g_dsp_ctx->dma_queue_bufs[i] = dma_buf; + } else { + free(dma_buf); + dma_buf = NULL; + g_dsp_ctx->dma_queue_bufs[i] = NULL; + wp = AEE_EFAILED; + break; + } + } else { + g_dsp_ctx->htp_ctx->dma_cached[i] = NULL; + g_dsp_ctx->dma_queue_bufs[i] = NULL; + wp = AEE_ENOMEMORY; + break; + } + + void * alias_buf = memalign(dma_align, dma_alias_size); + if (alias_buf && g_dsp_ctx->htp_ctx->dma_cached[i]) { + g_dsp_ctx->htp_ctx->dma[i] = dma_queue_alias_init(alias_buf, + g_dsp_ctx->htp_ctx->dma_cached[i], 1); + if (g_dsp_ctx->htp_ctx->dma[i]) { + g_dsp_ctx->dma_alias_bufs[i] = alias_buf; + } else { + free(alias_buf); + g_dsp_ctx->dma_alias_bufs[i] = NULL; + } + } else { + if (alias_buf) free(alias_buf); + g_dsp_ctx->htp_ctx->dma[i] = NULL; + g_dsp_ctx->dma_alias_bufs[i] = NULL; + } + } + printf("htp_ctx dma_queue created x%d (main+alias)\n", g_dsp_ctx->thread_counts); + if (wp != AEE_SUCCESS) { + GGMLHEXAGON_LOG_ERROR("dma_queue_init failed (wp=%d)", wp); + return wp; + } + } + + g_dsp_ctx->hexagon_power_ctx = (void *)(handle); + + // Test VTCM memory read/write (must ensure VTCM is available in cache mode) + if (g_dsp_ctx->vtcm_base != NULL) { + dsp_vtcm_acquire(); + uint8_t *weight = (uint8_t *)g_dsp_ctx->vtcm_base; + uint8_t *active = (uint8_t *)g_dsp_ctx->vtcm_base + 256; + // Write test patterns + memset(weight, 0xaa, 128); + memset(active, 0xbb, 128); + // Verify write + if (weight[0] == 0xaa && active[0] == 0xbb) { + GGMLHEXAGON_LOG_INFO("VTCM read/write test PASSED: weight[0]=0x%02x, active[0]=0x%02x", weight[0], active[0]); + } else { + GGMLHEXAGON_LOG_ERROR("VTCM read/write test FAILED: weight[0]=0x%02x, active[0]=0x%02x", weight[0], active[0]); + } + } else { + GGMLHEXAGON_LOG_WARN("VTCM not available, skipping VTCM test"); + } + + hap_probe_dsp(handle); + + GGMLHEXAGON_LOG_DEBUG("leave %s", __func__ ); + return AEE_SUCCESS; +} + +AEEResult ggml_dsp_register_rpcmem(remote_handle64 h, uint32_t ion_fd, uint32_t size_lo, uint32_t size_hi) { + g_dsp_ctx = (struct dsp_context *)h; + if (!g_dsp_ctx) return AEE_EBADPARM; + int32_t fd = (int32_t)ion_fd; + uint64_t size = ((uint64_t)size_hi << 32) | (uint64_t)size_lo; + + GGMLHEXAGON_LOG_INFO("[ION-REG] fd=%d, size=%llu bytes (%dMB)", + fd, (unsigned long long)size, (int32_t)(size >> 20)); + + int64_t t0_mmap = ggml_time_us(); +#if __HVX_ARCH__ > 73 + void * va = HAP_mmap2(NULL, (size_t)size, HAP_PROT_READ | HAP_PROT_WRITE, 0, fd, 0); +#else + void * va = HAP_mmap(NULL, (size_t)size, HAP_PROT_READ | HAP_PROT_WRITE, 0, fd, 0); +#endif + int64_t dt_mmap = ggml_time_us() - t0_mmap; + + if (va == NULL || va == (void *)-1) { + g_dsp_ctx->mempool_dsp_base = NULL; + GGMLHEXAGON_LOG_ERROR("[ION-REG] HAP_mmap2 FAILED: returned %p (fd=%d, size=%llu)", va, fd, (unsigned long long)size); + return AEE_EFAILED; + } + + g_dsp_ctx->mempool_dsp_base = va; + g_dsp_ctx->mempool_dsp_size = (size_t)size; + // Use FARF(ALWAYS) so the timing log is visible via adb logcat on all builds + FARF(ALWAYS, "[ION-REG] HAP_mmap2 OK: va=%p (fd=%d, size=%zuMB, time=%lld us)", va, fd, g_dsp_ctx->mempool_dsp_size / (1024*1024), (long long)dt_mmap); + + return AEE_SUCCESS; +} + +// Wake/suspend the work_queue + hmx_queue worker threads around a batch, +// mirroring htp/main.c htp_packet_callback. Without wakeup the workers stay in +// qurt_futex_wait and (depending on QuRT futex semantics) can miss seqn +// bumps; without hmx_queue_flush the batch can return while HMX descriptors +// are still in flight, so AP reads back incomplete dst tensors. +static void dsp_queues_wakeup(void) { + struct htp_context * htp = g_dsp_ctx->htp_ctx; + if (htp->work_queue) { + work_queue_wakeup(htp->work_queue); + } + if (htp->hmx_queue) { + hmx_queue_wakeup(htp->hmx_queue); + } +} + +static void dsp_queues_suspend(void) { + struct htp_context * htp = g_dsp_ctx->htp_ctx; + if (htp->hmx_queue) { + hmx_queue_suspend(htp->hmx_queue); + hmx_queue_flush(htp->hmx_queue); + } + if (htp->work_queue) { + work_queue_suspend(htp->work_queue); + } +} + +/* + * ION-based op-batch execution: FastRPC only passes 2 scalars (offset, size) - all data is in the mempool. + */ +AEEResult ggml_dsp_execute_batch(remote_handle64 h, uint32_t batch_offset, uint32_t batch_size) { + g_dsp_ctx = (struct dsp_context *)h; + if (!g_dsp_ctx) return AEE_EBADPARM; + if (g_dsp_ctx->mempool_dsp_base == NULL) { + GGMLHEXAGON_LOG_ERROR("ION base not registered"); + return AEE_EBADPARM; + } + + const char * base = (const char *)g_dsp_ctx->mempool_dsp_base; + + /* dsp_cache_mode config mode: batch_size == 0xFFFC. batch_offset encodes + * bits 0..3 : dsp_cache_mode (first-touch weight / prior-dst skip / bulk dst flush / selective bulk flush) + * bit 4 : reset weight_inval first-touch array (model reload) + * bit 16 : dsp_cache_trace_bit0 (1 = emit [DSP-CACHE-TRACE-BIT0] per bit 0 decision) + * bit 17 : dsp_cache_trace_bit1 (1 = emit [DSP-CACHE-TRACE-BIT1] per bit 1 decision) + * Pushed by AP at ggmlhexagon_init_cdsp() time. Bit definitions: + * bit 0 (0x1): first-touch weight tracking - INVAL_SRC_IF_NEEDED skips + * dcinva for repack weights (flags==2) once invalidated. + * bit 1 (0x2): skip dcinva for prior dst - INVAL_SRC_IF_NEEDED skips + * dcinva for activations (flags!=2) when [base,base+len) is + * fully contained in a dst range that DSP wrote earlier in + * this batch. Only effective when bit 2 is also on. + * bit 2 (0x4): bulk dst flush at batch end - per-op flush is suppressed; + * dst ranges collected/sort/merged/flushed once after loop. + * bit 3 (0x8): selective bulk flush - skip batch-end flush for + * dsts still consumed by a later op in this batch (pure + * intermediates). Only effective when bit 2 is also on. + * Mirrored dsts (flags&0x1) and final outputs always flush. + * bit 4 (0x10): reset weight_inval array - clear first-touch tracking + * so new model weights at reused ION addresses get properly + * invalidated after model unload/reload. */ + if (batch_size == 0xFFFC) { + g_dsp_ctx->dsp_cache_mode = batch_offset & 0xFu; + g_dsp_ctx->dsp_cache_trace_bit0 = (batch_offset >> 16) & 0x1u; + g_dsp_ctx->dsp_cache_trace_bit1 = (batch_offset >> 17) & 0x1u; + if (batch_offset & 0x10u) { + weight_inval_reset_all(); + } + GGMLHEXAGON_LOG_INFO("[DSP-CACHE-MODE] dsp_cache_mode=0x%x (bit0=first-touch=%d bit1=prior-dst=%d bit2=bulk-flush=%d bit3=selective=%d bit4=reset-inval=%d) trace0=%d trace1=%d", + g_dsp_ctx->dsp_cache_mode, + (g_dsp_ctx->dsp_cache_mode & 0x1) ? 1 : 0, + (g_dsp_ctx->dsp_cache_mode & 0x2) ? 1 : 0, + (g_dsp_ctx->dsp_cache_mode & 0x4) ? 1 : 0, + (g_dsp_ctx->dsp_cache_mode & 0x8) ? 1 : 0, + (batch_offset & 0x10u) ? 1 : 0, + g_dsp_ctx->dsp_cache_trace_bit0, + g_dsp_ctx->dsp_cache_trace_bit1); + return AEE_SUCCESS; + } + + /* Warmup mode: batch_size == 0xFFFB. + * AP calls this once after session init to warm up the FastRPC/ION path + * without doing any real compute. Just logs and returns. */ + if (batch_size == 0xFFFB) { + GGMLHEXAGON_LOG_INFO("[DSP-WARMUP] no-op warmup done"); + return AEE_SUCCESS; + } + + /* Normal batch execution */ +#if HEX_OP_PROF + const int64_t prof_batch_t0 = ggml_time_us(); +#endif + /* Invalidate DSP cache for the batch descriptor before reading. + * ION is non-coherent: AP reuses the mempool and writes a new batch + * at the same offset, so DSP must invalidate to fetch fresh data. + * Use dcinva (invalidate only) instead of dccleaninva (clean+invalidate): + * dccleaninva would write back stale DSP cache lines to DRAM, overwriting + * the fresh data AP just flushed via DC CVAC. */ +#if HEX_OP_PROF + const int64_t prof_hdr_t0 = ggml_time_us(); +#endif + ggml_dsp_cache_inval_range((void *)(base + batch_offset), batch_size); +#if HEX_OP_PROF + g_dsp_ctx->nonop_hdr_inval_us += (uint64_t)(ggml_time_us() - prof_hdr_t0); +#endif + const hex_batch_hdr * hdr = (const hex_batch_hdr *)(base + batch_offset); + + if (hdr->n_ops == 0 || hdr->n_tensors == 0) { + GGMLHEXAGON_LOG_ERROR("empty ion-batch: n_ops=%u n_tensors=%u", hdr->n_ops, hdr->n_tensors); + return AEE_EBADPARM; + } + if (hdr->n_tensors > DSP_OPT_MAX_TENSORS) { + GGMLHEXAGON_LOG_ERROR("n_tensors %u exceeds DSP_OPT_MAX_TENSORS %u", + hdr->n_tensors, DSP_OPT_MAX_TENSORS); + return AEE_EBADPARM; + } + if (hdr->n_ops > DSP_OPT_MAX_TENSORS * 4) { + GGMLHEXAGON_LOG_ERROR("n_ops %u exceeds limit (%u)", hdr->n_ops, DSP_OPT_MAX_TENSORS * 4); + return AEE_EBADPARM; + } + /* Validate offsets before dereferencing to guard against corrupted batch descriptors */ + if (hdr->ops_offset == 0 || hdr->tensors_offset == 0 || + hdr->ops_offset >= batch_size || hdr->tensors_offset >= batch_size || + hdr->ops_offset + hdr->n_ops * (uint32_t)sizeof(hex_op_desc) > batch_size || + hdr->tensors_offset + hdr->n_tensors * (uint32_t)sizeof(hex_tensor_desc) > batch_size) { + GGMLHEXAGON_LOG_ERROR("invalid batch layout: ops_off=%u tens_off=%u total=%u n_ops=%u n_tensors=%u", + hdr->ops_offset, hdr->tensors_offset, hdr->total_size, + hdr->n_ops, hdr->n_tensors); + return AEE_EBADPARM; + } + + const hex_op_desc * ops = (const hex_op_desc *)((const char *)hdr + hdr->ops_offset); + const hex_tensor_desc * tens = (const hex_tensor_desc *)((const char *)hdr + hdr->tensors_offset); + + /* Validate per-op src_idx/dst_idx against n_tensors to prevent + * OOB access into g_dsp_ctx->pre_dt/g_dsp_ctx->pre_ht/g_dsp_ctx->batch_tensor_needs_inval. */ + for (uint32_t i = 0; i < hdr->n_ops; i++) { + const hex_op_desc * op = &ops[i]; + for (int s = 0; s < HTP_OP_MAX_INPUTS; s++) { + if (op->src_idx[s] >= 0 && (uint32_t)op->src_idx[s] >= hdr->n_tensors) { + GGMLHEXAGON_LOG_ERROR("op %u: src_idx[%d]=%d exceeds n_tensors=%u", + i, s, op->src_idx[s], hdr->n_tensors); + return AEE_EBADPARM; + } + } + for (int d = 0; d < HTP_OP_MAX_OUTPUTS; d++) { + if (op->dst_idx[d] >= 0 && (uint32_t)op->dst_idx[d] >= hdr->n_tensors) { + GGMLHEXAGON_LOG_ERROR("op %u: dst_idx[%d]=%d exceeds n_tensors=%u", + i, d, op->dst_idx[d], hdr->n_tensors); + return AEE_EBADPARM; + } + } + } + + /* Validate per-tensor data ranges against the mempool mapping. Catches + * corrupted descriptors (e.g. stale offsets from the AP side) before + * they become wild reads/writes anywhere in DSP address space. */ + for (uint32_t ti = 0; ti < hdr->n_tensors; ti++) { + const uint32_t off = tens[ti].data_offset; + const uint32_t len = tens[ti].data_len; + if (off >= g_dsp_ctx->mempool_dsp_size || len > g_dsp_ctx->mempool_dsp_size - off) { + GGMLHEXAGON_LOG_ERROR("tensor %u: range out of bounds off=%u len=%u pool=%zu", + ti, off, len, (size_t)g_dsp_ctx->mempool_dsp_size); + return AEE_EBADPARM; + } + } + + /* VTCM is acquired once in ggml_dsp_open and held for the whole + * session (matches htp/main.c htp_packet_callback pattern). Per-batch + * acquire/release removed: see dsp_vtcm_acquire in ggml_dsp_open. */ + + GGMLHEXAGON_LOG_DEBUG("ion-batch: start n_ops=%u n_tensors=%u", hdr->n_ops, hdr->n_tensors); + + /* Reset per-batch dst trackers. + * - prior_dst_ranges is consulted by bit 1; the per-op dst tracker + * populates it when bit 2 is on. Resetting on bit 1 OR bit 2 keeps + * the list clean even if bit 1 is enabled without bit 2. + * - bulk_flush_ranges is populated only when bit 2 is on, but the + * list itself is harmless when empty (bulk_flush_all() early-returns + * if count==0). */ + if (g_dsp_ctx->dsp_cache_mode & (0x2 | 0x4)) { + prior_dst_reset_all(); + } + if (g_dsp_ctx->dsp_cache_mode & 0x4) { + bulk_flush_reset_all(); + } + + /* Pre-convert all tensors once: saves one hex_tensor_to_dsptensor() + * call per tensor per op. The per-op loop below references g_dsp_ctx->pre_dt + * by pointer (srcs) or copies + overrides op_params (dsts). */ + +#if HEX_OP_PROF + const int64_t prof_pre_t0 = ggml_time_us(); +#endif + for (uint32_t ti = 0; ti < hdr->n_tensors; ti++) { + hex_tensor_to_dsptensor(&tens[ti], base, &g_dsp_ctx->pre_dt[ti]); + hex_tensor_to_htp_tensor(&tens[ti], base, &g_dsp_ctx->pre_ht[ti]); + } +#if HEX_OP_PROF + g_dsp_ctx->nonop_preconvert_us += (uint64_t)(ggml_time_us() - prof_pre_t0); +#endif + + /* Reset per-batch invalidation tracking: all tensors start as + * "needs invalidation" (1). Set to 0 when invalidated; set back + * to 1 when written as dst (dirtied). */ + memset(g_dsp_ctx->batch_tensor_needs_inval, 1, hdr->n_tensors); + + /* bit 3: record each tensor's last consumer op index so the dst + * tracker can tell pure intermediates (last_use > producer op) + * from final outputs (never consumed later). */ + if (g_dsp_ctx->dsp_cache_mode & 0x8) { + memset(g_dsp_ctx->tensor_last_use_op, 0, hdr->n_tensors * sizeof(g_dsp_ctx->tensor_last_use_op[0])); + for (uint32_t oi = 0; oi < hdr->n_ops; oi++) { + for (int s = 0; s < HTP_OP_MAX_INPUTS; s++) { + const int32_t si = ops[oi].src_idx[s]; + if (si >= 0 && (uint32_t)si < hdr->n_tensors) { + g_dsp_ctx->tensor_last_use_op[si] = oi; + } + } + } + } + +#if HEX_OP_PROF + const int64_t prof_q_t0 = ggml_time_us(); +#endif + dsp_queues_wakeup(); +#if HEX_OP_PROF + g_dsp_ctx->nonop_queue_us += (uint64_t)(ggml_time_us() - prof_q_t0); +#endif + + for (uint32_t i = 0; i < hdr->n_ops; i++) { + const hex_op_desc * op = &ops[i]; + + /* srcs: reference pre-converted tensors directly */ + const dsptensor *src0_dt = &g_dsp_ctx->pre_dt[op->src_idx[0]]; + const dsptensor *src1_dt = (op->src_idx[1] >= 0) ? &g_dsp_ctx->pre_dt[op->src_idx[1]] : NULL; + const dsptensor *src2_dt = (op->src_idx[2] >= 0) ? &g_dsp_ctx->pre_dt[op->src_idx[2]] : NULL; + const dsptensor *src3_dt = (op->src_idx[3] >= 0) ? &g_dsp_ctx->pre_dt[op->src_idx[3]] : NULL; + const dsptensor *src4_dt = (op->src_idx[4] >= 0) ? &g_dsp_ctx->pre_dt[op->src_idx[4]] : NULL; + const dsptensor *src5_dt = (op->src_idx[5] >= 0) ? &g_dsp_ctx->pre_dt[op->src_idx[5]] : NULL; + + if (1 == g_dsp_ctx->dump_diag_info) { + if (src0_dt->data && src0_dt->data_len >= 16) { + const float * fv = (const float *)src0_dt->data; + GGMLHEXAGON_LOG_INFO("[DSP-DIAG] op%u src0 PRE-INVAL off=0x%x ptr=%p f32=[%.4f, %.4f, %.4f, %.4f]", + i, tens[op->src_idx[0]].data_offset, src0_dt->data, fv[0], fv[1], fv[2], fv[3]); + } + if (src1_dt && src1_dt->data && src1_dt->data_len >= 16 && src1_dt->type == 0) { + const float * fv = (const float *)src1_dt->data; + GGMLHEXAGON_LOG_INFO("[DSP-DIAG] op%u src1 off=0x%x ptr=%p f32=[%.4f, %.4f, %.4f, %.4f] ne=[%d,%d,%d,%d]", + i, tens[op->src_idx[1]].data_offset, src1_dt->data, fv[0], fv[1], fv[2], fv[3], + (int)src1_dt->ne[0], (int)src1_dt->ne[1], (int)src1_dt->ne[2], (int)src1_dt->ne[3]); + } + } + + /* dsts: copy from pre-converted, then override op_params with + * per-op params (node->op_params is correct; dst tensor's op_params + * can be zero or stale for in-place reuse ops like SCALE). + * Uses dsp_context-scoped buffers (g_dsp_ctx->dst_dt_buf/dst_dt_ptrs) + * to avoid per-op stack frame pressure. */ + for (int k = 0; k < HTP_OP_MAX_OUTPUTS; k++) { + g_dsp_ctx->dst_dt_ptrs[k] = NULL; + const int32_t di = op->dst_idx[k]; + if (di < 0) continue; + g_dsp_ctx->dst_dt_buf[k] = g_dsp_ctx->pre_dt[di]; + memcpy(g_dsp_ctx->dst_dt_buf[k].op_params, op->params, sizeof(g_dsp_ctx->dst_dt_buf[k].op_params)); + g_dsp_ctx->dst_dt_ptrs[k] = &g_dsp_ctx->dst_dt_buf[k]; + } + + /* Cache maintenance for non-coherent mempool memory. + * Per-batch tracking skips redundant dcinva when the same tensor is + * used as src by multiple ops. See dsp_cache_mode bit definitions + * at the batch_size==0xFFFC gate above. */ + INVAL_SRC_IF_NEEDED(i, 0, src0_dt, op->src_idx[0]); + if (src1_dt) INVAL_SRC_IF_NEEDED(i, 1, src1_dt, op->src_idx[1]); + if (src2_dt) INVAL_SRC_IF_NEEDED(i, 2, src2_dt, op->src_idx[2]); + if (src3_dt) INVAL_SRC_IF_NEEDED(i, 3, src3_dt, op->src_idx[3]); + if (src4_dt) INVAL_SRC_IF_NEEDED(i, 4, src4_dt, op->src_idx[4]); + if (src5_dt) INVAL_SRC_IF_NEEDED(i, 5, src5_dt, op->src_idx[5]); + + if (1 == g_dsp_ctx->dump_diag_info) { + if (src0_dt->data && src0_dt->data_len >= 16) { + const float * fv = (const float *)src0_dt->data; + float eps_f; + memcpy(&eps_f, g_dsp_ctx->dst_dt_buf[0].op_params, sizeof(float)); + GGMLHEXAGON_LOG_INFO("[DSP-DIAG] op%u src0 POST-INVAL off=0x%x ptr=%p f32=[%.4f, %.4f, %.4f, %.4f] eps=%f ne=[%d,%d,%d,%d]", + i, tens[op->src_idx[0]].data_offset, src0_dt->data, fv[0], fv[1], fv[2], fv[3], eps_f, + (int)src0_dt->ne[0], (int)src0_dt->ne[1], (int)src0_dt->ne[2], (int)src0_dt->ne[3]); + } + } + + GGMLHEXAGON_LOG_DEBUG("ion-batch: op %u/%u opc=%d", i, hdr->n_ops, op->opcode); + + // Translation layer: map GGML op to HTP op, build octx, call execute_op. + // For fused ops, AP sets htp_opcode directly (skip ggml_op_to_htp_op). + enum htp_op_code htp_op; + if (op->htp_opcode != 0) { + htp_op = (enum htp_op_code) op->htp_opcode; + } else if (ggml_op_to_htp_op(op->opcode, op->params, &htp_op) != 0) { + GGMLHEXAGON_LOG_ERROR("ion-op %u: unsupported opcode %d", i, op->opcode); + dsp_queues_suspend(); + return AEE_EUNSUPPORTED; + } + + GGMLHEXAGON_LOG_DEBUG("ion-op %u: htp_op=%u opcode=%d", i, htp_op, op->opcode); + + struct htp_ops_context octx; + + build_htp_octx(&octx, htp_op, op->params, op->kernel_params, + op->src_idx, op->dst_idx); + + if (htp_op == HTP_OP_MUL_MAT) { + const int32_t kp_kernel_type = octx.kernel_params[0]; + if (kp_kernel_type == 0) { + if (build_mm_kernel_params(&octx) != 0) { + dsp_queues_suspend(); + return AEE_EFAILED; + } + } + } + + if (htp_op == HTP_OP_FLASH_ATTN_EXT) { + const int32_t kp_kernel_type = octx.kernel_params[0]; + /* AP always precomputes FA params; kernel_type==UNSUPPORTED means + * the op was routed here against fa_select policy (e.g. direct + * graph_compute with fa_select=0). Reject instead of running it. */ + if (kp_kernel_type == HTP_FA_KERNEL_UNSUPPORTED) { + GGMLHEXAGON_LOG_ERROR("ion-op %u: FLASH_ATTN_EXT without precomputed params", i); + dsp_queues_suspend(); + return AEE_EUNSUPPORTED; + } + } + +#ifndef NDEBUG + /* F32 MUL_MAT diagnostic: dump src0 row 0/16, src1 row 0, dst[16] BEFORE execute_op. */ + if (htp_op == HTP_OP_MUL_MAT && src0_dt->type == 0 /*F32*/ && + src0_dt->data && src0_dt->data_len >= (size_t)(17 * 256) && + src1_dt && src1_dt->data && src1_dt->data_len >= 16 && + g_dsp_ctx->dst_dt_ptrs[0] && g_dsp_ctx->dst_dt_buf[0].data && g_dsp_ctx->dst_dt_buf[0].data_len >= (size_t)(17 * 4)) { + const float * s0 = (const float *) src0_dt->data; + const float * s1 = (const float *) src1_dt->data; + const float * dp = (const float *) g_dsp_ctx->dst_dt_buf[0].data; + const uint32_t s0_row16_off = src0_dt->nb[1] * 16 / 4; + GGMLHEXAGON_LOG_ERROR("[DSP-MM-PRE] op%u kp_type=%d s0r0=[%.4f,%.4f,%.4f,%.4f] s0r16=[%.4f,%.4f,%.4f,%.4f] s1r0=[%.4f,%.4f,%.4f,%.4f] dst16=[%.4f,%.4f,%.4f,%.4f] nb=[%u,%u,%u,%u] ne=[%u,%u,%u,%u]", + i, octx.kernel_params[0], + s0[0], s0[1], s0[2], s0[3], + s0[s0_row16_off+0], s0[s0_row16_off+1], s0[s0_row16_off+2], s0[s0_row16_off+3], + s1[0], s1[1], s1[2], s1[3], + dp[16], dp[17], dp[18], dp[19], + src0_dt->nb[0], src0_dt->nb[1], src0_dt->nb[2], src0_dt->nb[3], + src0_dt->ne[0], src0_dt->ne[1], src0_dt->ne[2], src0_dt->ne[3]); + /* ktype, pipe, mch, nch, nthr, nact, nhmx, npf, src1rs, vtcm_sz, + * vtcm_src0, vtcm_src1, vtcm_dst */ + GGMLHEXAGON_LOG_ERROR("[DSP-MM-KP] op%u ktype=%d pipe=%d mch=%d nch=%d nthr=%d nact=%d nhmx=%d npf=%d src1rs=%d vtcm_sz=%d src0_sz=%d src1_sz=%d dst_sz=%d", + i, octx.kernel_params[0], octx.kernel_params[1], octx.kernel_params[2], + octx.kernel_params[3], octx.kernel_params[4], octx.kernel_params[5], + octx.kernel_params[6], octx.kernel_params[7], octx.kernel_params[10], + octx.kernel_params[11], octx.kernel_params[12], octx.kernel_params[13], + octx.kernel_params[16]); + } +#endif + + int op_ret = execute_op(&octx); + +#ifndef NDEBUG + /* F32 MUL_MAT diagnostic: dump dst[0..3] and dst[16..19] AFTER execute_op. */ + if (htp_op == HTP_OP_MUL_MAT && src0_dt->type == 0 /*F32*/ && + g_dsp_ctx->dst_dt_ptrs[0] && g_dsp_ctx->dst_dt_buf[0].data && g_dsp_ctx->dst_dt_buf[0].data_len >= (size_t)(20 * 4)) { + const float * dp = (const float *) g_dsp_ctx->dst_dt_buf[0].data; + GGMLHEXAGON_LOG_ERROR("[DSP-MM-POST] op%u d[0..3]=[%.4f,%.4f,%.4f,%.4f] d[16..19]=[%.4f,%.4f,%.4f,%.4f]", + i, dp[0], dp[1], dp[2], dp[3], dp[16], dp[17], dp[18], dp[19]); + } +#endif + + // Clear spad refs (matches proc_op_req post-execute cleanup) + octx.src0_spad.src = NULL; + octx.src1_spad.src = NULL; + octx.src2_spad.src = NULL; + octx.src3_spad.src = NULL; + octx.dst_spad.src = NULL; + + if (op_ret != HTP_STATUS_OK) { + const char * st_name = + (op_ret == HTP_STATUS_INTERNAL_ERR) ? "INTERNAL_ERR" : + (op_ret == HTP_STATUS_NO_SUPPORT) ? "NO_SUPPORT" : + (op_ret == HTP_STATUS_INVAL_PARAMS) ? "INVAL_PARAMS" : + (op_ret == HTP_STATUS_VTCM_TOO_SMALL) ? "VTCM_TOO_SMALL" : "UNKNOWN"; + GGMLHEXAGON_LOG_ERROR("ion-op %u: execute_op returned %d/%s (htp_op=%d)", + i, op_ret, st_name, htp_op); + dsp_queues_suspend(); + return AEE_EFAILED; + } + + GGMLHEXAGON_LOG_DEBUG("ion-batch: op %u done", i); + + /* bit 2: bulk dst flush at batch end. + * Also mark dst tensors as dirty so they get re-invalidated + * when used as src later in the same batch. */ +#if HEX_OP_PROF + const int64_t prof_dst_t0 = ggml_time_us(); +#endif + for (int k = 0; k < HTP_OP_MAX_OUTPUTS; k++) { + if (!g_dsp_ctx->dst_dt_ptrs[k]) continue; + if (g_dsp_ctx->dsp_cache_mode & 0x4) { + /* prior_dst ranges are only consumed by bit 1; avoid dead + * work when bit 2 is enabled without bit 1. */ + if (g_dsp_ctx->dsp_cache_mode & 0x2) { + prior_dst_add(g_dsp_ctx->dst_dt_buf[k].data, g_dsp_ctx->dst_dt_buf[k].data_len, + op->dst_idx[k], htp_op); + } + /* bit 3: pure intermediates are read back from DSP L2 by + * their in-batch consumers, so their batch-end flush is + * dead traffic. Mirrored dsts (flags&0x1) are read by AP + * after the batch and must always be flushed. */ + const int32_t di = op->dst_idx[k]; + const bool skip_flush = + (g_dsp_ctx->dsp_cache_mode & 0x8) && + g_dsp_ctx->tensor_last_use_op[di] > i && + !(g_dsp_ctx->dst_dt_buf[k].flags & 0x1); + if (!skip_flush) { + bulk_flush_add(g_dsp_ctx->dst_dt_buf[k].data, g_dsp_ctx->dst_dt_buf[k].data_len); + } + } else { + ggml_dsp_cache_flush_range(g_dsp_ctx->dst_dt_buf[k].data, g_dsp_ctx->dst_dt_buf[k].data_len); + } + /* Mark dst tensor as dirty: next time it's used as src, it must be re-invalidated. */ + g_dsp_ctx->batch_tensor_needs_inval[op->dst_idx[k]] = 1; + if (g_dsp_ctx->dsp_cache_mode & 0x1) { + weight_inval_unmark(g_dsp_ctx->dst_dt_buf[k].data); + } + } +#if HEX_OP_PROF + g_dsp_ctx->nonop_dst_track_us += (uint64_t)(ggml_time_us() - prof_dst_t0); +#endif + + if (1 == g_dsp_ctx->dump_diag_info) { + /* DSP-side DIAG: dump first 4 f32 values from each dst output */ + for (int k = 0; k < HTP_OP_MAX_OUTPUTS; k++) { + if (g_dsp_ctx->dst_dt_ptrs[k] && g_dsp_ctx->dst_dt_buf[k].data && g_dsp_ctx->dst_dt_buf[k].data_len >= 16) { + const float * fv = (const float *)g_dsp_ctx->dst_dt_buf[k].data; + GGMLHEXAGON_LOG_INFO("[DSP-DIAG] op%u dst[%d] off=0x%x ptr=%p f32=[%.4f, %.4f, %.4f, %.4f]", + i, k, tens[op->dst_idx[k]].data_offset, g_dsp_ctx->dst_dt_buf[k].data, fv[0], fv[1], fv[2], fv[3]); + } + } + } + } + + GGMLHEXAGON_LOG_DEBUG("ion-batch: all %u ops done", hdr->n_ops); + +#if HEX_OP_PROF + const int64_t prof_qs_t0 = ggml_time_us(); +#endif + dsp_queues_suspend(); +#if HEX_OP_PROF + g_dsp_ctx->nonop_queue_us += (uint64_t)(ggml_time_us() - prof_qs_t0); +#endif + + /* bit 2: bulk dst flush. Sort + merge collected ranges, then flush once + * per merged region. Flushes happen here (after all ops in the batch + * finished) so AP reads see fresh DRAM. */ +#if HEX_OP_PROF + const int64_t prof_bf_t0 = ggml_time_us(); +#endif + if (g_dsp_ctx->dsp_cache_mode & 0x4) { + bulk_flush_all(); + } +#if HEX_OP_PROF + g_dsp_ctx->nonop_bulk_flush_us += (uint64_t)(ggml_time_us() - prof_bf_t0); +#endif + + /* Read-probe one word of the last op's first output before replying, + * forcing the memory pipeline to observe completed writes; result is + * discarded and the offset is bounds-checked above. */ + __asm__ __volatile__("" ::: "memory"); + if (hdr->n_ops > 0 && ops[hdr->n_ops - 1].dst_idx[0] >= 0 && + (uint32_t)ops[hdr->n_ops - 1].dst_idx[0] < hdr->n_tensors) { + uint32_t last_off = tens[ops[hdr->n_ops - 1].dst_idx[0]].data_offset; + if (last_off < g_dsp_ctx->mempool_dsp_size && g_dsp_ctx->mempool_dsp_size - last_off > 4) + (void) *(volatile const int *)(base + last_off); + } + __asm__ __volatile__("" ::: "memory"); + +#if HEX_OP_PROF + /* Per-op profiler: print accumulated cum/count/avg every N batches so + * log volume stays bounded (one multi-line dump per interval, not per op). */ + g_dsp_ctx->op_prof_batch_wall_us += (uint64_t)(ggml_time_us() - prof_batch_t0); + g_dsp_ctx->op_prof_batch_count++; + if ((g_dsp_ctx->op_prof_batch_count % HEX_OP_PROF_DUMP_INTERVAL) == 0) { + char tag[32]; + snprintf(tag, sizeof(tag), "batch#%lu", (unsigned long)g_dsp_ctx->op_prof_batch_count); + dump_op_prof(tag); + /* whole-batch wall vs sum of per-op buckets: the delta is the DSP-side + * non-op overhead (descriptor inval, tensor pre-convert, per-op src + * dcinva, bulk dst flush, queue wakeup/suspend). */ + uint64_t op_sum = 0; + for (unsigned int b = 0; b < HEX_OP_PROF_BUCKETS; b++) op_sum += g_dsp_ctx->op_prof_dur_us[b]; + FARF(ERROR, "[OP-PROF] %s batch-wall cum=%llu us (avg=%llu) op-sum cum=%llu us (avg=%llu) non-op avg=%lld us/batch", + tag, + (unsigned long long) g_dsp_ctx->op_prof_batch_wall_us, + (unsigned long long) (g_dsp_ctx->op_prof_batch_wall_us / g_dsp_ctx->op_prof_batch_count), + (unsigned long long) op_sum, + (unsigned long long) (op_sum / g_dsp_ctx->op_prof_batch_count), + (long long) ((int64_t) (g_dsp_ctx->op_prof_batch_wall_us - op_sum) / (int64_t) g_dsp_ctx->op_prof_batch_count)); + const uint64_t nbc = g_dsp_ctx->op_prof_batch_count; + FARF(ERROR, "[OP-PROF-NONOP] %s hdr=%llu pre=%llu w-inv=%llu(%lluMB) a-inv=%llu(%lluMB) dst=%llu bulk=%llu queue=%llu us/batch", + tag, + (unsigned long long) (g_dsp_ctx->nonop_hdr_inval_us / nbc), + (unsigned long long) (g_dsp_ctx->nonop_preconvert_us / nbc), + (unsigned long long) (g_dsp_ctx->nonop_w_inval_us / nbc), + (unsigned long long) (g_dsp_ctx->nonop_w_inval_bytes / nbc / (1024 * 1024)), + (unsigned long long) (g_dsp_ctx->nonop_a_inval_us / nbc), + (unsigned long long) (g_dsp_ctx->nonop_a_inval_bytes / nbc / (1024 * 1024)), + (unsigned long long) (g_dsp_ctx->nonop_dst_track_us / nbc), + (unsigned long long) (g_dsp_ctx->nonop_bulk_flush_us / nbc), + (unsigned long long) (g_dsp_ctx->nonop_queue_us / nbc)); + } +#endif + + return AEE_SUCCESS; +} diff --git a/ggml/src/ggml-hexagon/htp/ggml_dsp.idl b/ggml/src/ggml-hexagon/htp/ggml_dsp.idl new file mode 100644 index 000000000000..ebfae4140b22 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/ggml_dsp.idl @@ -0,0 +1,10 @@ +#include "AEEStdDef.idl" +#include "remote.idl" + +const string IDL_VERSION = "0.0.2"; + +interface ggml_dsp : remote_handle64 { + AEEResult setclocks(in int32 diag_info, in int32 requested_thread_counts, rout int32 actual_thread_counts); + AEEResult register_rpcmem(in uint32 ion_fd, in uint32 size_lo, in uint32 size_hi); + AEEResult execute_batch(in uint32 batch_offset, in uint32 batch_size); +}; diff --git a/scripts/build-run-ggmlhexagon-android.sh b/scripts/build-run-ggmlhexagon-android.sh new file mode 100755 index 000000000000..a7e539baa1d9 --- /dev/null +++ b/scripts/build-run-ggmlhexagon-android.sh @@ -0,0 +1,1488 @@ +#!/usr/bin/env bash +# +set -e + +######## part-1: public macros & vars ######## + +PWD=`pwd` +PROJECT_HOME_PATH=`pwd` +PROJECT_ROOT_PATH=${PROJECT_HOME_PATH} +HOST_CPU_COUNTS=`cat /proc/cpuinfo | grep "processor" | wc | awk '{print int($1)}'` + +VERBOSE=OFF +VERBOSE=ON + +#running path on Android phone +REMOTE_PATH=/data/local/tmp + +#path of built artifacts +LOCAL_BUILD_DIR=${PROJECT_ROOT_PATH}/out/ggmlhexagon-android + +TOOLCHAIN_PATH=${PROJECT_ROOT_PATH}/prebuilts + +#Android NDK can be found at: +#https://developer.android.com/ndk/downloads +ANDROID_PLATFORM=android-34 +ANDROID_NDK_VERSION=r29 +ANDROID_NDK_NAME=android-ndk-${ANDROID_NDK_VERSION} +ANDROID_NDK_FULLNAME=${ANDROID_NDK_NAME}-linux.zip +ANDROID_NDK=${TOOLCHAIN_PATH}/${ANDROID_NDK_NAME} + +# --- Define NDK paths based on the absolute SDK path --- +NDK_TOOLCHAIN_SYSROOT_INCLUDE_PATH="${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include" +NDK_TOOLCHAIN_SYSROOT_ARM64_LIB_PATH="${ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/aarch64-linux-android" + +#OpenCL Headers can be found at: +#https://https://github.com/KhronosGroup/OpenCL-Headers +OPENCL_SDK_URL=https://github.com/KhronosGroup/OpenCL-Headers +OPENCL_SDK_PATH=${TOOLCHAIN_PATH}/OpenCL_SDK +OPENCL_HEADERS_PATH=${OPENCL_SDK_PATH}/OpenCL-Headers + +#fully Qualcomm Hexagon SDK can be found at https://developer.qualcomm.com/software/hexagon-dsp-sdk/tools. +#fully Hexagon SDK must be obtained with Qualcomm Developer Account and follow PKLA&ECA. +HEXAGON_SDK_VERSION=6.6.0.0 +HEXAGON_TOOLS_VERSION=19.0.07 +HEXAGON_SDK_PATH=${TOOLCHAIN_PATH}/Hexagon_SDK/${HEXAGON_SDK_VERSION} +HEXAGON_TOOLS_PATH=${HEXAGON_SDK_PATH}/tools/HEXAGON_Tools/${HEXAGON_TOOLS_VERSION} + +#supported htp arch version: +#v73 --- Snapdragon 8 Gen2 +#v75 --- Snapdragon 8 Gen3 +#v79 --- Snapdragon 8 Elite(aka 8 Gen4) +#v81 --- Snapdragon 8 Elite Gen5(aka 8 Gen5) + +HTP_ARCH_VERSIONS="v73 v75 v79 v81" + +# default HTP_ARCH +HTP_ARCH_VERSION=${HTP_ARCH_VERSIONS%% *} + +######## part-2: prompt and LLM models ######## + +#2.9 GiB, default model, will be downloadded automatically via this script when running this script at the first time +GGUF_MODEL_NAME=/sdcard/gemma-4-E2B-it-Q4_0.gguf + +# Model aliases for quick testing of multiple models +# Usage: ./scripts/build-run-ggmlhexagon-android.sh run_llamacli +# qwen3-2b -> Qwen3.5-2B-Q4_0.gguf +# qwen3-9b -> Qwen3.5-9B-Q4_0.gguf +# gemma4-e2b -> gemma-4-E2B-it-Q4_0.gguf +# gemma4-e4b -> gemma-4-E4B_q4_0-it.gguf +# qwen1 -> qwen1_5-1_8b-chat-q4_0.gguf +# llama3 -> Llama-3.2-1B-Instruct-Q4_0.gguf +# nanbeige-3b -> Nanbeige_Nanbeige4.2-3B-Q4_0.gguf +# minicpm5-1b -> minicpm5-1b-q4_0.gguf +# (default) -> gemma-4-E2B-it-Q4_0.gguf +# nanbeige-3b-q80 -> Nanbeige_Nanbeige4.2-3B-Q8_0.gguf +# minicpm5-1b-q80 -> MiniCPM5-1B-Q8_0.gguf +function resolve_model_name() +{ + case "$1" in + qwen3-2b) echo "/sdcard/Qwen3.5-2B-Q4_0.gguf" ;; + qwen3-9b) echo "/sdcard/Qwen3.5-9B-Q4_0.gguf" ;; + gemma4-e2b) echo "/sdcard/gemma-4-E2B-it-Q4_0.gguf" ;; + gemma4-e4b) echo "/sdcard/gemma-4-E4B_q4_0-it.gguf" ;; + qwen1) echo "/sdcard/qwen1_5-1_8b-chat-q4_0.gguf" ;; + llama3) echo "/sdcard/Llama-3.2-1B-Instruct-Q4_0.gguf" ;; + nanbeige-3b) echo "/sdcard/Nanbeige_Nanbeige4.2-3B-Q4_0.gguf";; + nanbeige-3b-q80) echo "/sdcard/Nanbeige_Nanbeige4.2-3B-Q8_0.gguf";; + minicpm5-1b) echo "/sdcard/minicpm5-1b-q4_0.gguf";; + minicpm5-1b-q80) echo "/sdcard/MiniCPM5-1B-Q8_0.gguf";; + *) echo "" ; return 1 ;; + esac +} + +PROMPT_STRING="Hello, good morning, you are a powerful domain expert and know many things, now pls help to introduce the movie Once Upon a Time in America briefly, pls pay attention short then 1000 words\n" + +running_params=" -ngl 99 -t 6 -n 256 --ctx-size 8192 --ubatch-size 64 --poll 1000 --no-warmup --load-mode none -fa on --jinja -st" + +######## part-3: utilities and functions ######## + +function dump_vars() +{ + echo -e "ANDROID_NDK: ${ANDROID_NDK}" + echo -e "HEXAGON_SDK_PATH: ${HEXAGON_SDK_PATH}" +} + + +function show_pwd() +{ + echo -e "current working path:$(pwd)\n" +} + + +function check_command_in_host() +{ + set +e + cmd=$1 + if command -v ${cmd} > /dev/null 2>&1; then + printf "${cmd} is available on host machine\n" > /dev/null + else + printf "${cmd} not exist on host machine, pls install command line utility ${cmd} firstly and accordingly\n" + exit 1 + fi + set -e +} + + +function check_commands_in_host() +{ + check_command_in_host wget + check_command_in_host xzcat + check_command_in_host adb + check_command_in_host md5sum + check_command_in_host ninja +} + + +function check_android_phone() +{ + local device_raw + device_raw=$(adb devices 2>/dev/null | grep -v "List of devices" | awk 'NF>0') + + if [[ -z "$device_raw" ]]; then + adb kill-server >/dev/null 2>&1 + sleep 0.1 + adb start-server >/dev/null 2>&1 + device_raw=$(adb devices 2>/dev/null | grep -v "List of devices" | awk 'NF>0') + if [[ -z "$device_raw" ]]; then + echo "No Android device detected." + echo "Please check if phone is connected properly.Exiting" + exit 1 + fi + fi + + if echo "$device_raw" | grep -q "no permissions"; then + echo "Device detected but has NO PERMISSIONS." + echo "Please check if phone is connected properly.Exiting" + exit 1 + fi + + if echo "$device_raw" | grep -q "unauthorized"; then + echo "Device detected but UNAUTHORIZED." + echo "Please check if phone is connected properly.Exiting" + exit 1 + fi + + if echo "$device_raw" | grep -q "offline"; then + echo "Device is OFFLINE." + echo "Please check if phone is connected properly.Exiting" + exit 1 + fi + + if echo "$device_raw" | awk '{print $2}' | grep -qx "device"; then + local sn=$(echo "$device_raw" | awk '{print $1}') + echo "Android device connected successfully: $sn" + return 0 + fi + + echo "Unknown device error." + echo "Please check if phone is connected properly.Exiting" + exit 1 +} + + +#TODO:refine this function +function check_and_download_hexagon_sdk() +{ + is_hexagon_llvm_exist=1 + if [ ! -f ${TOOLCHAIN_PATH}/Hexagon_SDK/${HEXAGON_SDK_VERSION}/tools/HEXAGON_Tools/${HEXAGON_TOOLS_VERSION}/NOTICE.txt ]; then + echo -e "${TEXT_RED}minimal-hexagon-sdk not exist...${TEXT_RESET}\n" + is_hexagon_llvm_exist=0 + fi + + if [ ${is_hexagon_llvm_exist} -eq 0 ]; then + mkdir -p ${TOOLCHAIN_PATH}/Hexagon_SDK/ + if [ -f ${TOOLCHAIN_PATH}/Hexagon_SDK/hexagon-sdk-v6.6.0.0-amd64-lnx.tar.xz ]; then + echo -e "hexagon-sdk-v6.6.0.0-amd64-lnx.tar.xz already exist\n" + else + echo -e "begin downloading hexagon-sdk-v6.6.0.0-amd64-lnx.tar.xz \n" + wget --no-config --quiet --show-progress -O ${TOOLCHAIN_PATH}/Hexagon_SDK/hexagon-sdk-v6.6.0.0-amd64-lnx.tar.xz https://github.com/snapdragon-toolchain/hexagon-sdk/releases/download/v6.6.0.0/hexagon-sdk-v6.6.0.0-amd64-lnx.tar.xz + if [ $? -ne 0 ]; then + printf "failed to download hexagon-sdk-v6.6.0.0-amd64-lnx.tar.xz\n" + exit 1 + fi + fi + + echo -e "begin decompressing hexagon-sdk-v6.6.0.0-amd64-lnx.tar.xz \n" + xzcat ${TOOLCHAIN_PATH}/Hexagon_SDK/hexagon-sdk-v6.6.0.0-amd64-lnx.tar.xz | tar -C ${TOOLCHAIN_PATH}/Hexagon_SDK/ -xf - + if [ $? -ne 0 ]; then + printf "failed to decompress hexagon-sdk-v6.6.0.0-amd64-lnx.tar.xz\n" + exit 1 + fi + printf "install minimal-hexagon-sdk successfully\n\n" + fi + + if [ ! -d ${HEXAGON_SDK_PATH} ]; then + echo -e "HEXAGON_SDK_PATH ${HEXAGON_SDK_PATH} not exist, pls install it accordingly...\n" + exit 0 + else + printf "Qualcomm Hexagon SDK already exist:${HEXAGON_SDK_PATH} \n\n" > /dev/null + fi +} + + +#not mandatory for ggml-hexagon +function check_and_download_opencl_sdk() +{ + is_opencl_sdk_exist=1 + + if [ ! -d ${OPENCL_SDK_PATH} ]; then + echo -e "OPENCL_SDK_PATH ${OPENCL_SDK_PATH} not exist, download it from ${OPENCL_SDK_URL}...\n" + is_opencl_sdk_exist=0 + fi + if [ ! -f ${NDK_TOOLCHAIN_SYSROOT_ARM64_LIB_PATH}/libOpenCL.so ]; then + echo -e "${NDK_TOOLCHAIN_SYSROOT_ARM64_LIB_PATH}/libOpenCL.so not exist...\n" + is_opencl_sdk_exist=0 + fi + + if [ ${is_opencl_sdk_exist} -eq 0 ]; then + mkdir -p ${OPENCL_SDK_PATH} + cd ${OPENCL_SDK_PATH} + + if [ ! -d OpenCL-Headers ]; then + echo "Cloning OpenCL-Headers..." + git clone https://github.com/KhronosGroup/OpenCL-Headers + if [ $? -ne 0 ]; then + printf "failed to download OpenCL-Headers to %s \n" "${OPENCL_SDK_PATH}" + exit 1 + fi + fi + cd ${TOOLCHAIN_PATH}/OpenCL_SDK/OpenCL-Headers + printf "Copying OpenCL Headers to Android NDK sysroot include: ${NDK_TOOLCHAIN_SYSROOT_INCLUDE_PATH}" + mkdir -p ${NDK_TOOLCHAIN_SYSROOT_INCLUDE_PATH} + /bin/cp -r -fv CL ${NDK_TOOLCHAIN_SYSROOT_INCLUDE_PATH} + + cd ${TOOLCHAIN_PATH}/OpenCL_SDK + if [ ! -d OpenCL-ICD-Loader ]; then + echo "Cloning OpenCL-ICD-Loader..." + git clone https://github.com/KhronosGroup/OpenCL-ICD-Loader + if [ $? -ne 0 ]; then + printf "failed to download OpenCL-ICD-Loader to %s \n" "${OPENCL_SDK_PATH}" + exit 1 + fi + fi + cd ${TOOLCHAIN_PATH}/OpenCL_SDK/OpenCL-ICD-Loader + mkdir -p build + cd build + cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=latest -DANDROID_STL=c++_shared -DOPENCL_ICD_LOADER_HEADERS_DIR=${NDK_TOOLCHAIN_SYSROOT_INCLUDE_PATH} + echo "Building OpenCL-ICD-Loader with ninjia..." + ninja + if [ $? -ne 0 ]; then + printf "failed to build OpenCL-ICD-Loader\n" + exit 1 + fi + mkdir -p ${NDK_TOOLCHAIN_SYSROOT_ARM64_LIB_PATH} + /bin/cp -fv libOpenCL.so ${NDK_TOOLCHAIN_SYSROOT_ARM64_LIB_PATH} + + echo "OpenCL components setup complete" + echo "OpenCL Headers are in: ${NDK_TOOLCHAIN_SYSROOT_INCLUDE_PATH}/CL" + echo "libOpenCL.so is in: ${NDK_TOOLCHAIN_SYSROOT_ARM64_LIB_PATH}/libOpenCL.so" + + cd ${PROJECT_ROOT_PATH} + else + printf "OpenCL SDK already exist: ${OPENCL_SDK_PATH} \n\n" > /dev/null + fi +} + + +function check_and_download_ndk() +{ + is_android_ndk_exist=1 + + if [ ! -d ${ANDROID_NDK} ]; then + is_android_ndk_exist=0 + fi + + if [ ! -f ${ANDROID_NDK}/build/cmake/android.toolchain.cmake ]; then + is_android_ndk_exist=0 + fi + + if [ ${is_android_ndk_exist} -eq 0 ]; then + + if [ ! -f ${TOOLCHAIN_PATH}/${ANDROID_NDK_FULLNAME} ]; then + wget --no-config --quiet --show-progress -O ${TOOLCHAIN_PATH}/${ANDROID_NDK_FULLNAME} https://dl.google.com/android/repository/${ANDROID_NDK_FULLNAME} + fi + + cd ${TOOLCHAIN_PATH} + unzip ${ANDROID_NDK_FULLNAME} + + if [ $? -ne 0 ]; then + printf "failed to download Android NDK to %s \n" "${ANDROID_NDK}" + exit 1 + fi + cd ${PROJECT_ROOT_PATH} + + printf "Android NDK saved to ${ANDROID_NDK} \n\n" + else + printf "Android NDK already exist: ${ANDROID_NDK} \n\n" > /dev/null + fi +} + + +#build the mempool/FastRPC-invoke ggml-hexagon backend (default) +function build_arm64 +{ + export CCACHE_DIR=${PROJECT_ROOT_PATH}/.ccache + + # clear dspqueue skels left by a prior build_dspqueue, else detect_build_type() + # misreports hexagon-dspqueue + rm -f ${LOCAL_BUILD_DIR}/ggml/src/ggml-hexagon/libggml-htp-*.so + + #ARMv8.7a+i8mm CPU tuning flags, moved here from CMakeLists.txt to keep it aligned with upstream master + local arm_cpu_flags="-march=armv8.7a+fp16+dotprod+i8mm -fvectorize -ffp-model=fast -fno-finite-math-only -flto -D_GNU_SOURCE" + + local extra_flags="${arm_cpu_flags}" + + /bin/cp -fv ${PROJECT_ROOT_PATH}/docs/backend/snapdragon/CMakeUserPresets.json . + + cmake -H. -B${LOCAL_BUILD_DIR} -DCMAKE_BUILD_TYPE=Release -DGGML_OPENMP=OFF -DGGML_OPENCL=OFF -DGGML_CCACHE=ON -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=latest -DGGML_HEXAGON=ON -DLLAMA_CURL=OFF -DGGML_LLAMAFILE=ON -DGGML_HEXAGON_USE_MEMPOOL=ON -DHEXAGON_SDK_ROOT=${HEXAGON_SDK_PATH} -DHEXAGON_TOOLS_ROOT=${HEXAGON_TOOLS_PATH} -DCMAKE_C_FLAGS="${extra_flags}" -DCMAKE_CXX_FLAGS="${extra_flags}" -DCMAKE_VERBOSE_MAKEFILE:BOOL=${VERBOSE} -DGGML_USE_HEXAGON=ON -DLLAMA_BUILD_TESTS=ON -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_BUILD_APP=OFF -DLLAMA_BUILD_UI=ON -DLLAMA_USE_PREBUILT_UI=OFF -DLLAMA_OPENSSL=OFF --preset arm64-android-snapdragon-release + cmake --build ${LOCAL_BUILD_DIR} + #upload the new libggml-htp.so (mempool variant) on device side + prepare_fastrpc_skels + #push AP-side libs too: libggml-hexagon.so embeds the regenerated FastRPC stub + #which MUST stay in sync with the DSP skel signature, otherwise FastRPC args + #get misaligned (root cause of the 8gen3 garbled-output regression). + update_ggml_libs + commit_so_file_md5 ${LOCAL_BUILD_DIR}/bin/libggml-cpu.so + if [ -f ${LOCAL_BUILD_DIR}/bin/libggml-hexagon.so ]; then + commit_so_file_md5 ${LOCAL_BUILD_DIR}/bin/libggml-hexagon.so + # backup for AB testing: mempool(FastRPC) AP-side libs + DSP skels + # libggml-hexagon.so leaks different libc++ symbols between the mempool/dspqueue + # variants, so all transitively-linked libs (libggml, libllama, libllama-common, + # *-impl) must be swapped together to avoid symbol lookup failures at runtime + mkdir -p ${PROJECT_ROOT_PATH}/out/ab-test + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/libggml-hexagon.so ${PROJECT_ROOT_PATH}/out/ab-test/libggml-hexagon-fastrpc.so + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/libggml.so ${PROJECT_ROOT_PATH}/out/ab-test/libggml-fastrpc.so + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/libllama.so ${PROJECT_ROOT_PATH}/out/ab-test/libllama-fastrpc.so + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/libllama-common.so ${PROJECT_ROOT_PATH}/out/ab-test/libllama-common-fastrpc.so + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/libllama-completion-impl.so ${PROJECT_ROOT_PATH}/out/ab-test/libllama-completion-impl-fastrpc.so + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/libllama-server-impl.so ${PROJECT_ROOT_PATH}/out/ab-test/libllama-server-impl-fastrpc.so + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/libmtmd.so ${PROJECT_ROOT_PATH}/out/ab-test/libmtmd-fastrpc.so + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/libllama-bench-impl.so ${PROJECT_ROOT_PATH}/out/ab-test/libllama-bench-impl-fastrpc.so + for skel in ${LOCAL_BUILD_DIR}/bin/libggml-htp-v*.so; do + [ -f "$skel" ] || continue + /bin/cp -fv "$skel" ${PROJECT_ROOT_PATH}/out/ab-test/$(basename "$skel" .so)-fastrpc.so + done + # libggml-opencl.so is optional (GGML_OPENCL=OFF by default); back up if present + if [ -f ${LOCAL_BUILD_DIR}/bin/libggml-opencl.so ]; then + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/libggml-opencl.so ${PROJECT_ROOT_PATH}/out/ab-test/libggml-opencl-fastrpc.so + fi + fi + show_pwd + + /bin/rm -f CMakeUserPresets.json +} + + +#build the dspqueue ggml-hexagon backend for performance comparison +function build_arm64_dspqueue +{ + #make AI Agent happy + export CCACHE_DIR=${PROJECT_ROOT_PATH}/.ccache_dspqueue + + rm -f ${LOCAL_BUILD_DIR}/.ab_test_runtime + + # clear mempool skels left by a prior fastrpc build, else detect_build_type() + # misreports hexagon-fastrpc + rm -f ${LOCAL_BUILD_DIR}/bin/libggml-htp-*.so + + /bin/cp -fv ${PROJECT_ROOT_PATH}/docs/backend/snapdragon/CMakeUserPresets.json . + + cmake -H. -B${LOCAL_BUILD_DIR} -DCMAKE_BUILD_TYPE=Release -DGGML_OPENMP=OFF -DGGML_OPENCL=OFF -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=latest -DGGML_HEXAGON=ON -DLLAMA_CURL=OFF -DGGML_LLAMAFILE=ON -DHEXAGON_SDK_ROOT=${HEXAGON_SDK_PATH} -DHEXAGON_TOOLS_ROOT=${HEXAGON_TOOLS_PATH} --preset arm64-android-snapdragon-release -DCMAKE_VERBOSE_MAKEFILE:BOOL=${VERBOSE} + cmake --build ${LOCAL_BUILD_DIR} + #upload the new libggml-htp.so (dspqueue variant) on device side + prepare_dspqueue_skels + #push AP-side libs too: dspqueue build also needs to sync runtime libs + update_ggml_libs + # backup for AB testing: dspqueue AP-side libs + DSP skels + if [ -f ${LOCAL_BUILD_DIR}/bin/libggml-hexagon.so ]; then + mkdir -p ${PROJECT_ROOT_PATH}/out/ab-test + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/libggml-hexagon.so ${PROJECT_ROOT_PATH}/out/ab-test/libggml-hexagon-dspqueue.so + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/libggml.so ${PROJECT_ROOT_PATH}/out/ab-test/libggml-dspqueue.so + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/libllama.so ${PROJECT_ROOT_PATH}/out/ab-test/libllama-dspqueue.so + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/libllama-common.so ${PROJECT_ROOT_PATH}/out/ab-test/libllama-common-dspqueue.so + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/libllama-completion-impl.so ${PROJECT_ROOT_PATH}/out/ab-test/libllama-completion-impl-dspqueue.so + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/libllama-server-impl.so ${PROJECT_ROOT_PATH}/out/ab-test/libllama-server-impl-dspqueue.so + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/libmtmd.so ${PROJECT_ROOT_PATH}/out/ab-test/libmtmd-dspqueue.so + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/libllama-bench-impl.so ${PROJECT_ROOT_PATH}/out/ab-test/libllama-bench-impl-dspqueue.so + for skel in ${LOCAL_BUILD_DIR}/ggml/src/ggml-hexagon/libggml-htp-v*.so; do + [ -f "$skel" ] || continue + /bin/cp -fv "$skel" ${PROJECT_ROOT_PATH}/out/ab-test/$(basename "$skel" .so)-dspqueue.so + done + # libggml-opencl.so is optional (GGML_OPENCL=OFF by default); back up if present + if [ -f ${LOCAL_BUILD_DIR}/bin/libggml-opencl.so ]; then + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/libggml-opencl.so ${PROJECT_ROOT_PATH}/out/ab-test/libggml-opencl-dspqueue.so + fi + fi + show_pwd + + /bin/rm -f CMakeUserPresets.json + + echo "run following command to see the performance of the dspqueue ggml-hexagon backend" + echo "./scripts/build-run-ggmlhexagon-android.sh run_llamacli" + echo "./scripts/build-run-ggmlhexagon-android.sh run_llamabench" +} + + +#build Android CPU-only reference (no ggml-hexagon) for correctness check and troubleshooting trick issues +function build_armcpu() +{ + export CCACHE_DIR=${PROJECT_ROOT_PATH}/.ccache_cpu + + #ARMv8.7a+i8mm CPU tuning flags, moved here from CMakeLists.txt to keep it aligned with upstream master + local arm_cpu_flags="-march=armv8.7a+fp16+dotprod+i8mm -fvectorize -ffp-model=fast -fno-finite-math-only -flto -D_GNU_SOURCE" + + cmake -H. -B${LOCAL_BUILD_DIR} -DCMAKE_BUILD_TYPE=Release -DGGML_OPENMP=OFF -DGGML_CCACHE=ON -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=latest -DGGML_HEXAGON=OFF -DLLAMA_CURL=OFF -DGGML_LLAMAFILE=ON -DCMAKE_C_FLAGS="${arm_cpu_flags}" -DCMAKE_CXX_FLAGS="${arm_cpu_flags}" -DCMAKE_VERBOSE_MAKEFILE:BOOL=${VERBOSE} + cd ${LOCAL_BUILD_DIR} + # use cmake --build so it matches whatever generator the cache was configured with + cmake --build ${LOCAL_BUILD_DIR} -j${HOST_CPU_COUNTS} + #remove stale hexagon artifacts from previous hexagon builds to ensure CPU-only runtime + rm -f ${LOCAL_BUILD_DIR}/bin/libggml-hexagon.so + rm -f ${LOCAL_BUILD_DIR}/bin/libggml-htp-*.so + # legacy name from before the skel unification; clean up leftovers + rm -f ${LOCAL_BUILD_DIR}/bin/libggmldsp-skel-*.so + # also clear dspqueue skels left in the source dir by a prior build_dspqueue, else + # detect_build_type() falls back to them and misreports hexagon-dspqueue + rm -f ${LOCAL_BUILD_DIR}/ggml/src/ggml-hexagon/libggml-htp-*.so + # backup CPU-only AP libs for AB switching (symmetric with build/build_dspqueue) + # note: do NOT use "-cpu" suffix - libggml-cpu.so is the CPU backend impl, a + # different lib from libggml.so (the core). use "-cpuonly" to avoid collision. + mkdir -p ${PROJECT_ROOT_PATH}/out/ab-test + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/libggml.so ${PROJECT_ROOT_PATH}/out/ab-test/libggml-cpuonly.so + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/libllama.so ${PROJECT_ROOT_PATH}/out/ab-test/libllama-cpuonly.so + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/libllama-common.so ${PROJECT_ROOT_PATH}/out/ab-test/libllama-common-cpuonly.so + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/libllama-completion-impl.so ${PROJECT_ROOT_PATH}/out/ab-test/libllama-completion-impl-cpuonly.so + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/libllama-server-impl.so ${PROJECT_ROOT_PATH}/out/ab-test/libllama-server-impl-cpuonly.so + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/libmtmd.so ${PROJECT_ROOT_PATH}/out/ab-test/libmtmd-cpuonly.so + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/libllama-bench-impl.so ${PROJECT_ROOT_PATH}/out/ab-test/libllama-bench-impl-cpuonly.so + + # fix issue CANNOT LINK EXECUTABLE "/data/local/tmp/llama-completion": library "libggml-hexagon.so" not found: needed by main executable + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/llama-bench ${PROJECT_ROOT_PATH}/out/ab-test/llama-bench-cpuonly + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/llama-completion ${PROJECT_ROOT_PATH}/out/ab-test/llama-completion-cpuonly + /bin/cp -fv ${LOCAL_BUILD_DIR}/bin/llama-server ${PROJECT_ROOT_PATH}/out/ab-test/llama-server-cpuonly + + show_pwd +} + + +function remove_temp_dir() +{ + if [ -d ${LOCAL_BUILD_DIR} ]; then + echo "remove ${LOCAL_BUILD_DIR} directory" + rm -rf ${LOCAL_BUILD_DIR} + fi +} + + +function update_cfg() +{ + if [ -f ${PROJECT_ROOT_PATH}/scripts/ggml-hexagon.cfg ]; then + adb push ${PROJECT_ROOT_PATH}/scripts/ggml-hexagon.cfg ${REMOTE_PATH}/ + fi +} + + +function build_ggml_hexagon() +{ + show_pwd + check_and_download_ndk + check_and_download_opencl_sdk + check_and_download_hexagon_sdk + dump_vars + remove_temp_dir + rm -f ${LOCAL_BUILD_DIR}/.ab_test_runtime + build_arm64 +} + + +function build_ggml_hexagon_dspqueue() +{ + show_pwd + check_and_download_ndk + check_and_download_opencl_sdk + check_and_download_hexagon_sdk + dump_vars + remove_temp_dir + build_arm64_dspqueue +} + + +#push dspqueue-variant DSP skels (libggml-htp-vXX.so) to the device +function prepare_dspqueue_skels() +{ + for ver in ${HTP_ARCH_VERSIONS}; do + case "$ver" in + v73 | v75 | v79 | v81) + echo "adb push ${LOCAL_BUILD_DIR}/ggml/src/ggml-hexagon/libggml-htp-${ver}.so ${REMOTE_PATH}/libggml-htp-${ver}.so" + adb push ${LOCAL_BUILD_DIR}/ggml/src/ggml-hexagon/libggml-htp-${ver}.so ${REMOTE_PATH}/libggml-htp-${ver}.so + ;; + *) + show_usage + exit 1 + ;; + esac + done +} + + +#push mempool/FastRPC-variant DSP skels (libggml-htp-vXX.so) to the device +function prepare_fastrpc_skels() +{ + if [ -f ${PROJECT_ROOT_PATH}/scripts/ggml-hexagon.cfg ]; then + adb push ${PROJECT_ROOT_PATH}/scripts/ggml-hexagon.cfg ${REMOTE_PATH}/ggml-hexagon.cfg + fi + for ver in ${HTP_ARCH_VERSIONS}; do + if [ -f ${LOCAL_BUILD_DIR}/bin/libggml-htp-${ver}.so ]; then + echo "adb push ${LOCAL_BUILD_DIR}/bin/libggml-htp-${ver}.so ${REMOTE_PATH}/libggml-htp-${ver}.so" + adb push ${LOCAL_BUILD_DIR}/bin/libggml-htp-${ver}.so ${REMOTE_PATH}/libggml-htp-${ver}.so + fi + done +} + + +function check_and_download_model() +{ + set +e + + model_name=$1 + model_url=$2 + + adb shell ls /sdcard/${model_name} >/dev/null 2>&1 + if [ $? -eq 0 ]; then + printf "the prebuild LLM model ${model_name} already exist on Android phone\n" > /dev/null + else + printf "the prebuild LLM model ${model_name} not exist on Android phone\n" + printf "downloading from ${model_url}\n" + wget --no-config --quiet --show-progress -O ${PROJECT_ROOT_PATH}/models/${model_name} ${model_url} + adb push ${PROJECT_ROOT_PATH}/models/${model_name} /sdcard/ + fi + + set -e +} + + +function check_prebuilt_models() +{ + set +e + + #1.12 GiB + check_and_download_model qwen1_5-1_8b-chat-q4_0.gguf https://huggingface.co/Qwen/Qwen1.5-1.8B-Chat-GGUF/resolve/main/qwen1_5-1_8b-chat-q4_0.gguf + + #1.2 GiB + check_and_download_model Qwen3.5-2B-Q4_0.gguf https://huggingface.co/unsloth/Qwen3.5-2B-GGUF/resolve/main/Qwen3.5-2B-Q4_0.gguf + + #5.1 GiB + check_and_download_model Qwen3.5-9B-Q4_0.gguf https://huggingface.co/unsloth/Qwen3.5-9B-GGUF/resolve/main/Qwen3.5-9B-Q4_0.gguf + + #2.9 GiB + check_and_download_model gemma-4-E2B-it-Q4_0.gguf https://huggingface.co/unsloth/gemma-4-E2B-it-GGUF/resolve/main/gemma-4-E2B-it-Q4_0.gguf + + #4.9 GiB + check_and_download_model gemma-4-E4B_q4_0-it.gguf https://huggingface.co/google/gemma-4-E4B-it-qat-q4_0-gguf/resolve/main/gemma-4-E4B_q4_0-it.gguf + + #737 MiB + check_and_download_model Llama-3.2-1B-Instruct-Q4_0.gguf https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_0.gguf + + #2.4 GiB + check_and_download_model Nanbeige_Nanbeige4.2-3B-Q4_0.gguf https://huggingface.com/bartowski/Nanbeige_Nanbeige4.2-3B-GGUF/resolve/main/Nanbeige_Nanbeige4.2-3B-Q4_0.gguf + + #4.2 GiB + #check_and_download_model Nanbeige_Nanbeige4.2-3B-Q8_0.gguf https://huggingface.co/bartowski/Nanbeige_Nanbeige4.2-3B-GGUF/resolve/main/Nanbeige_Nanbeige4.2-3B-Q8_0.gguf + + #1.1 GiB + #check_and_download_model MiniCPM5-1B-Q8_0.gguf https://huggingface.co/openbmb/MiniCPM5-1B-GGUF/resolve/main/MiniCPM5-1B-Q8_0.gguf + + #635 MiB + check_and_download_model minicpm5-1b-q4_0.gguf https://huggingface.co/Elmermoreno/MiniCPM5-1B-Q4_0-GGUF/resolve/main/minicpm5-1b-q4_0.gguf + set -e +} + + +# ============================================================================== +# Return codes: +# 0 = NO changes +# 1 = FILE CHANGED +# ============================================================================== +function is_so_file_changed() { + set +e + local so_file="$1" + local md5_file="${so_file}.md5" + + # check if .so exists + if [ ! -f "$so_file" ]; then + echo "ERROR: File not found: $so_file" + return 1 + fi + + # get current MD5 + local current_md5 + current_md5=$(md5sum "$so_file" | awk '{print $1}') + + # FIRST RUN: no MD5 file -> save it, return CHANGED + if [ ! -f "$md5_file" ]; then + echo "$current_md5" > "$md5_file" + echo "Initialized MD5 for $so_file" + return 1 + fi + + # read previous MD5 + local last_md5 + last_md5=$(cat "$md5_file") + + # compare + if [ "$current_md5" = "$last_md5" ]; then + # NO CHANGE + return 0 + else + # CHANGED -> update MD5 + echo "$current_md5" > "$md5_file" + return 1 + fi +} + +# Persist the current MD5 of a .so to its .md5 cache file. Call this ONLY after +# the file has been successfully pushed to the device. +function commit_so_file_md5() { + local so_file="$1" + local md5_file="${so_file}.md5" + if [ -f "$so_file" ]; then + md5sum "$so_file" | awk '{print $1}' > "$md5_file" + fi +} + + +# Push AP-side libs (libggml-*.so, libllama-*.so) from bin/ to device. +# AP-only: does NOT push DSP skels (libggml-htp-*.so). +# Does NOT switch backend - DSP skels already on device stay as-is. +# Use update_fastrpc_libs / update_dspqueue_libs for a full backend switch (AP + DSP). +# Gotcha: bin/ reflects the last build (fastrpc or dspqueue); pushing fastrpc AP libs +# while dspqueue DSP skels are still on device leaves an AP/DSP mismatch. +function update_ggml_libs() +{ + #adb push ${LOCAL_BUILD_DIR}/bin/*.so ${REMOTE_PATH}/ + adb push ${LOCAL_BUILD_DIR}/bin/libggml-base.so ${REMOTE_PATH}/ + adb push ${LOCAL_BUILD_DIR}/bin/libggml-cpu.so ${REMOTE_PATH}/ + #libggml-hexagon.so only exists in hexagon builds, not in CPU-only builds + if [ -f ${LOCAL_BUILD_DIR}/bin/libggml-hexagon.so ]; then + adb push ${LOCAL_BUILD_DIR}/bin/libggml-hexagon.so ${REMOTE_PATH}/ + fi + #libggml-opencl.so is optional (GGML_OPENCL=OFF by default) + if [ -f ${LOCAL_BUILD_DIR}/bin/libggml-opencl.so ]; then + adb push ${LOCAL_BUILD_DIR}/bin/libggml-opencl.so ${REMOTE_PATH}/ + fi + adb push ${LOCAL_BUILD_DIR}/bin/libggml.so ${REMOTE_PATH}/ + adb push ${LOCAL_BUILD_DIR}/bin/libllama-common.so ${REMOTE_PATH}/ + adb push ${LOCAL_BUILD_DIR}/bin/libllama-completion-impl.so ${REMOTE_PATH}/ + adb push ${LOCAL_BUILD_DIR}/bin/libllama-server-impl.so ${REMOTE_PATH}/ + adb push ${LOCAL_BUILD_DIR}/bin/libmtmd.so ${REMOTE_PATH}/ + adb push ${LOCAL_BUILD_DIR}/bin/libllama-bench-impl.so ${REMOTE_PATH}/ + adb push ${LOCAL_BUILD_DIR}/bin/libllama.so ${REMOTE_PATH}/ +} + + +#push mempool/FastRPC runtime .so from out/ab-test/ to device, renaming *-fastrpc.so to canonical names +function update_fastrpc_libs() +{ + local ab_test_dir=${PROJECT_ROOT_PATH}/out/ab-test + if [ ! -f ${ab_test_dir}/libggml-hexagon-fastrpc.so ]; then + echo "ERROR: ${ab_test_dir}/libggml-hexagon-fastrpc.so not found." + echo "Run '$0 build' first to populate AB test backups." + exit 1 + fi + adb push ${ab_test_dir}/libggml-hexagon-fastrpc.so ${REMOTE_PATH}/libggml-hexagon.so + adb push ${ab_test_dir}/libggml-fastrpc.so ${REMOTE_PATH}/libggml.so + adb push ${ab_test_dir}/libllama-fastrpc.so ${REMOTE_PATH}/libllama.so + adb push ${ab_test_dir}/libllama-common-fastrpc.so ${REMOTE_PATH}/libllama-common.so + adb push ${ab_test_dir}/libllama-completion-impl-fastrpc.so ${REMOTE_PATH}/libllama-completion-impl.so + adb push ${ab_test_dir}/libllama-server-impl-fastrpc.so ${REMOTE_PATH}/libllama-server-impl.so + adb push ${ab_test_dir}/libmtmd-fastrpc.so ${REMOTE_PATH}/libmtmd.so + adb push ${ab_test_dir}/libllama-bench-impl-fastrpc.so ${REMOTE_PATH}/libllama-bench-impl.so + # skels push as canonical libggml-htp-vXX.so, overwriting any dspqueue skels + for skel in ${ab_test_dir}/libggml-htp-v*-fastrpc.so; do + [ -f "$skel" ] || continue + adb push "$skel" ${REMOTE_PATH}/$(basename "$skel" -fastrpc.so).so + done + # libggml-opencl.so is optional (GGML_OPENCL=OFF by default) + if [ -f ${ab_test_dir}/libggml-opencl-fastrpc.so ]; then + adb push ${ab_test_dir}/libggml-opencl-fastrpc.so ${REMOTE_PATH}/libggml-opencl.so + else + adb shell "rm -f ${REMOTE_PATH}/libggml-opencl.so" + adb shell "rm -f ${REMOTE_PATH}/libggml-vulkan.so" + fi + # legacy name from before the skel unification; clean up leftovers + adb shell "rm -f ${REMOTE_PATH}/libggmldsp-skel-*.so" + echo "fastrpc" > ${LOCAL_BUILD_DIR}/.ab_test_runtime + echo "mempool/FastRPC runtime .so pushed to device." +} + + +#push dspqueue runtime .so from out/ab-test/ to device, renaming *-dspqueue.so to canonical names +function update_dspqueue_libs() +{ + local ab_test_dir=${PROJECT_ROOT_PATH}/out/ab-test + if [ ! -f ${ab_test_dir}/libggml-hexagon-dspqueue.so ]; then + echo "ERROR: ${ab_test_dir}/libggml-hexagon-dspqueue.so not found." + echo "Run '$0 build_dspqueue' first to populate AB test backups." + exit 1 + fi + adb push ${ab_test_dir}/libggml-hexagon-dspqueue.so ${REMOTE_PATH}/libggml-hexagon.so + adb push ${ab_test_dir}/libggml-dspqueue.so ${REMOTE_PATH}/libggml.so + adb push ${ab_test_dir}/libllama-dspqueue.so ${REMOTE_PATH}/libllama.so + adb push ${ab_test_dir}/libllama-common-dspqueue.so ${REMOTE_PATH}/libllama-common.so + adb push ${ab_test_dir}/libllama-completion-impl-dspqueue.so ${REMOTE_PATH}/libllama-completion-impl.so + adb push ${ab_test_dir}/libllama-server-impl-dspqueue.so ${REMOTE_PATH}/libllama-server-impl.so + adb push ${ab_test_dir}/libmtmd-dspqueue.so ${REMOTE_PATH}/libmtmd.so + adb push ${ab_test_dir}/libllama-bench-impl-dspqueue.so ${REMOTE_PATH}/libllama-bench-impl.so + # skels push as canonical libggml-htp-vXX.so, overwriting any fastrpc skels + for skel in ${ab_test_dir}/libggml-htp-v*-dspqueue.so; do + [ -f "$skel" ] || continue + adb push "$skel" ${REMOTE_PATH}/$(basename "$skel" -dspqueue.so).so + done + # libggml-opencl.so is optional (GGML_OPENCL=OFF by default) + if [ -f ${ab_test_dir}/libggml-opencl-dspqueue.so ]; then + adb push ${ab_test_dir}/libggml-opencl-dspqueue.so ${REMOTE_PATH}/libggml-opencl.so + else + adb shell "rm -f ${REMOTE_PATH}/libggml-opencl.so" + adb shell "rm -f ${REMOTE_PATH}/libggml-vulkan.so" + fi + # legacy name from before the skel unification; clean up leftovers + adb shell "rm -f ${REMOTE_PATH}/libggmldsp-skel-*.so" + echo "dspqueue" > ${LOCAL_BUILD_DIR}/.ab_test_runtime + echo "dspqueue runtime .so pushed to device." +} + + +#push CPU-only runtime .so from out/ab-test/ to device, renaming *-cpuonly.so to canonical names +function update_cpu_libs() +{ + local ab_test_dir=${PROJECT_ROOT_PATH}/out/ab-test + if [ ! -f ${ab_test_dir}/libggml-cpuonly.so ]; then + echo "ERROR: ${ab_test_dir}/libggml-cpuonly.so not found." + echo "Run '$0 build_armcpu' first to populate AB test backups." + exit 1 + fi + adb push ${ab_test_dir}/libggml-cpuonly.so ${REMOTE_PATH}/libggml.so + adb push ${ab_test_dir}/libllama-cpuonly.so ${REMOTE_PATH}/libllama.so + adb push ${ab_test_dir}/libllama-common-cpuonly.so ${REMOTE_PATH}/libllama-common.so + adb push ${ab_test_dir}/libllama-completion-impl-cpuonly.so ${REMOTE_PATH}/libllama-completion-impl.so + adb push ${ab_test_dir}/libllama-server-impl-cpuonly.so ${REMOTE_PATH}/libllama-server-impl.so + adb push ${ab_test_dir}/libmtmd-cpuonly.so ${REMOTE_PATH}/libmtmd.so + adb push ${ab_test_dir}/libllama-bench-impl-cpuonly.so ${REMOTE_PATH}/libllama-bench-impl.so + + # fix issue CANNOT LINK EXECUTABLE "/data/local/tmp/llama-completion": library "libggml-hexagon.so" not found: needed by main executable + # because there is a different linker procedure for Android‑CPU‑only builds + /bin/cp -f ${ab_test_dir}/llama-bench-cpuonly ${LOCAL_BUILD_DIR}/bin/llama-bench + /bin/cp -f ${ab_test_dir}/llama-completion-cpuonly ${LOCAL_BUILD_DIR}/bin/llama-completion + /bin/cp -f ${ab_test_dir}/llama-server-cpuonly ${LOCAL_BUILD_DIR}/bin/llama-server + + adb push ${ab_test_dir}/llama-bench-cpuonly ${REMOTE_PATH}/llama-bench + adb push ${ab_test_dir}/llama-completion-cpuonly ${REMOTE_PATH}/llama-completion + adb push ${ab_test_dir}/llama-server-cpuonly ${REMOTE_PATH}/llama-server + + # libggml-base.so / libggml-cpu.so are shared across builds, device-side kept as-is + # libggml-opencl.so is optional (GGML_OPENCL=OFF by default) + adb shell "rm -f ${REMOTE_PATH}/libggml-opencl.so" + adb shell "rm -f ${REMOTE_PATH}/libggml-vulkan.so" + adb shell "rm -f ${REMOTE_PATH}/libggml-hexagon.so" + adb shell "rm -f ${REMOTE_PATH}/libggmldsp-skel-*.so" + adb shell "rm -f ${REMOTE_PATH}/libggml-htp-*.so" + echo "cpu" > ${LOCAL_BUILD_DIR}/.ab_test_runtime + echo "CPU-only runtime .so pushed to device." +} + + +#detect build type from build output: hexagon-fastrpc, hexagon-dspqueue, or cpu-only +#fastrpc build puts skels in bin/; dspqueue build puts them in ggml/src/ggml-hexagon/ +function detect_build_type() +{ + if [ -f ${LOCAL_BUILD_DIR}/bin/libggml-hexagon.so ]; then + if ls ${LOCAL_BUILD_DIR}/bin/libggml-htp-*.so 1>/dev/null 2>&1; then + echo "hexagon-fastrpc" + else + echo "hexagon-dspqueue" + fi + elif ls ${LOCAL_BUILD_DIR}/ggml/src/ggml-hexagon/libggml-htp-*.so 1>/dev/null 2>&1; then + echo "hexagon-dspqueue" + else + echo "cpu-only" + fi +} + + +function prepare_run_on_phone() +{ + if [ $# != 1 ]; then + print "invalid param" + return + fi + program=$1 + + update_cfg + + check_prebuilt_models + + # AB test mode: if update_fastrpc_libs/update_dspqueue_libs/update_cpu_libs set the marker, + # skip lib/skel push - user has manually set up the runtime. + local ab_test_marker="${LOCAL_BUILD_DIR}/.ab_test_runtime" + if [ -f "${ab_test_marker}" ]; then + local ab_runtime=$(cat "${ab_test_marker}") + printf "AB test mode active: runtime='%s' (skipping lib/skel push)\n\n" "${ab_runtime}" + adb push ${LOCAL_BUILD_DIR}/bin/${program} ${REMOTE_PATH}/ + adb shell ls -l ${REMOTE_PATH}/libggml-*.so + adb shell chmod +x ${REMOTE_PATH}/${program} + adb shell "rm -f /data/local/tmp/${program}.farf" + adb shell "touch /data/local/tmp/${program}.farf" + adb shell "echo 0x1c > /data/local/tmp/${program}.farf" + return + fi + + local current_build_type + current_build_type=$(detect_build_type) + + local last_build_type_file="${LOCAL_BUILD_DIR}/.last_deployed_build_type" + local last_build_type="" + if [ -f "${last_build_type_file}" ]; then + last_build_type=$(cat "${last_build_type_file}") + fi + + if [ "${current_build_type}" != "${last_build_type}" ]; then + printf "build type changed: '%s' -> '%s', force update ggml libs\n\n" "${last_build_type}" "${current_build_type}" + update_ggml_libs + commit_so_file_md5 ${LOCAL_BUILD_DIR}/bin/libggml-cpu.so + if [ -f ${LOCAL_BUILD_DIR}/bin/libggml-hexagon.so ]; then + commit_so_file_md5 ${LOCAL_BUILD_DIR}/bin/libggml-hexagon.so + fi + echo "${current_build_type}" > "${last_build_type_file}" + else + local need_update=0 + is_so_file_changed ${LOCAL_BUILD_DIR}/bin/libggml-cpu.so + if [ $? -eq 0 ]; then + printf "${LOCAL_BUILD_DIR}/bin/libggml-cpu.so not changed\n" + #reuse cached/uploaded ggml runtime libs on device side to avoid time-consuming task on host side + else + printf "${LOCAL_BUILD_DIR}/bin/libggml-cpu.so has changed or first check\n" + need_update=1 + fi + if [ -f ${LOCAL_BUILD_DIR}/bin/libggml-hexagon.so ]; then + is_so_file_changed ${LOCAL_BUILD_DIR}/bin/libggml-hexagon.so + if [ $? -ne 0 ]; then + printf "${LOCAL_BUILD_DIR}/bin/libggml-hexagon.so has changed or first check\n" + need_update=1 + else + printf "${LOCAL_BUILD_DIR}/bin/libggml-hexagon.so not changed\n" + fi + fi + if [ ${need_update} -eq 0 ]; then + #host-side MD5 matches cache, but verify libs actually exist on device + #(user may have wiped /data/local/tmp manually, leaving host-side MD5 cache stale) + if ! adb shell ls ${REMOTE_PATH}/libggml-cpu.so >/dev/null 2>&1; then + printf "device-side libggml-cpu.so missing (maybe /data/local/tmp was wiped), force update ggml libs\n\n" + need_update=1 + fi + fi + if [ ${need_update} -eq 0 ]; then + printf "reuse cached/uploaded ggml runtime libs on device side\n\n" + else + #upload ggml runtime libs to Android phone + update_ggml_libs + commit_so_file_md5 ${LOCAL_BUILD_DIR}/bin/libggml-cpu.so + if [ -f ${LOCAL_BUILD_DIR}/bin/libggml-hexagon.so ]; then + commit_so_file_md5 ${LOCAL_BUILD_DIR}/bin/libggml-hexagon.so + fi + fi + fi + + #deploy/cleanup backend-specific libs per build type + case "${current_build_type}" in + hexagon-fastrpc) + # skels push as libggml-htp-vXX.so and overwrite any dspqueue skels + prepare_fastrpc_skels + # legacy name from before the skel unification; clean up leftovers + adb shell rm -f ${REMOTE_PATH}/libggmldsp-skel-*.so + ;; + hexagon-dspqueue) + # skels push as libggml-htp-vXX.so and overwrite any fastrpc skels + prepare_dspqueue_skels + adb shell rm -f ${REMOTE_PATH}/libggmldsp-skel-*.so + ;; + cpu-only) + adb shell rm -f ${REMOTE_PATH}/libggml-hexagon.so + adb shell rm -f ${REMOTE_PATH}/libggmldsp-skel-*.so + adb shell rm -f ${REMOTE_PATH}/libggml-htp-*.so + adb shell rm -f ${REMOTE_PATH}/libggml-opencl.so + adb shell rm -f ${REMOTE_PATH}/libggml-vulkan.so + ;; + esac + + adb push ${LOCAL_BUILD_DIR}/bin/${program} ${REMOTE_PATH}/ + + adb shell ls -l ${REMOTE_PATH}/libggml-*.so + + adb shell chmod +x ${REMOTE_PATH}/${program} + + # configuration for cDSP's logcat + # FARF bits: 0x01=LOW 0x02=MEDIUM 0x04=HIGH 0x08=ERROR 0x10=FATAL + # 0x1f = ALL (default; chatty: HAP_compute_res logs ~6x per RPC, 26K+/test) + # 0x1c = HIGH+ERROR+FATAL (drop LOW+MEDIUM verbose spam; keep diag) + # 0x18 = ERROR+FATAL (only errors) + # 0x00 = silent + adb shell "rm -f /data/local/tmp/${program}.farf" + adb shell "touch /data/local/tmp/${program}.farf" + adb shell "echo 0x1c > /data/local/tmp/${program}.farf" + #observe cDSP's log + #adb logcat | grep "CDSP0" +} + + +function run_llamaversion() +{ + prepare_run_on_phone llama-cli + + adb shell "cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && export GGML_HEXAGON_OPPOLL=1 \ + && ${REMOTE_PATH}/llama-completion --version" +} + + +function run_llamacli() +{ + local model_name="" + local model_path="" + + if [ $# -ge 1 ]; then + model_name="$1" + model_path=$(resolve_model_name "$model_name") + if [ -z "$model_path" ]; then + echo "ERROR: unknown model alias '$model_name'. Valid aliases: qwen3-2b, qwen3-9b, gemma4-e2b, gemma4-e4b, qwen1, llama3" + exit 1 + fi + else + model_path="${GGUF_MODEL_NAME}" + fi + + prepare_run_on_phone llama-completion + + #GGML_HEXAGON_OPPOLL is only effective for the dspqueue variant, doesn't apply to the mempool/FastRPC variant + echo "adb shell \"cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && export GGML_HEXAGON_OPPOLL=1 \ + && ${REMOTE_PATH}/llama-completion ${running_params} -m ${model_path} -p \"${PROMPT_STRING}\"" + adb shell "cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && export GGML_HEXAGON_OPPOLL=1 \ + && ${REMOTE_PATH}/llama-completion ${running_params} -m ${model_path} -p \"${PROMPT_STRING}\"" + +} + + +function run_llamacli_all() +{ + local models=("qwen1" "minicpm5-1b" "llama3" "qwen3-2b" "gemma4-e2b" "nanbeige-3b" "gemma4-e4b" "qwen3-9b") + + local total=${#models[@]} + local count=0 + + echo "==============================================" + echo " Batch inference test: ${#models[@]} models = ${total} tests" + echo "==============================================" + + for model in "${models[@]}"; do + count=$(( count + 1 )) + echo "" + echo "--- [${count}/${total}] model=${model} ---" + run_llamacli "${model}" + done + + echo "" + echo "==============================================" + echo " Batch inference test complete: ${total} tests done" + echo "==============================================" +} + + +function run_llamabench() +{ + local model_name="" + local model_path="" + + if [ $# -ge 1 ]; then + model_name="$1" + model_path=$(resolve_model_name "$model_name") + if [ -z "$model_path" ]; then + echo "ERROR: unknown model alias '$model_name'. Valid aliases: qwen3-2b, qwen3-9b, gemma4-e2b, gemma4-e4b, qwen1, llama3" + exit 1 + fi + else + model_path="${GGUF_MODEL_NAME}" + fi + + prepare_run_on_phone llama-bench + + #GGML_HEXAGON_OPPOLL is only effective for the dspqueue variant, doesn't apply to the mempool/FastRPC variant + echo "adb shell \"cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && export GGML_HEXAGON_OPPOLL=1 \ + && ${REMOTE_PATH}/llama-bench -t 6 --poll 1000 -ngl 99 -fa 1 --ubatch-size 1024 -p 200,500,800,1024 -n 128 -m ${model_path}\"" + + adb shell "cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && export GGML_HEXAGON_OPPOLL=1 \ + && ${REMOTE_PATH}/llama-bench -t 6 --poll 1000 -ngl 99 -fa 1 --ubatch-size 1024 -p 200,500,800,1024 -n 128 -m ${model_path}" +} + + +function run_abtest() +{ + # mempool/FastRPC vs dspqueue performance comparison test. + # Requires out/ab-test/ populated (run 'build' then 'build_dspqueue' first). + # Usage: run_abtest [rounds] [model_alias] + # rounds: default 3 + # model_alias: default gemma4-e2b + # + # Example: + # $0 run_abtest + # $0 run_abtest 5 + # $0 run_abtest 3 qwen3-2b + # $0 run_abtest 2>&1 | tee log_abtest_$(date +%Y%m%d-%H%M%S).txt + + local rounds=3 + local model_path="${GGUF_MODEL_NAME}" + local ab_test_dir=${PROJECT_ROOT_PATH}/out/ab-test + + if [ $# -ge 1 ]; then + rounds=$1 + fi + if [ $# -ge 2 ]; then + local model_alias="$2" + model_path=$(resolve_model_name "$model_alias") + if [ -z "$model_path" ]; then + echo "ERROR: unknown model alias '$model_alias'. Valid aliases: qwen3-2b, qwen3-9b, gemma4-e2b, gemma4-e4b, qwen1, llama3" + exit 1 + fi + fi + + # sanity check: verify out/ab-test/ has all required .so from both builds + local missing="" + local fastrpc_libs="libggml-hexagon-fastrpc.so libggml-fastrpc.so libllama-fastrpc.so libllama-common-fastrpc.so libllama-completion-impl-fastrpc.so libllama-server-impl-fastrpc.so libmtmd-fastrpc.so libllama-bench-impl-fastrpc.so" + local dspqueue_libs="libggml-hexagon-dspqueue.so libggml-dspqueue.so libllama-dspqueue.so libllama-common-dspqueue.so libllama-completion-impl-dspqueue.so libllama-server-impl-dspqueue.so libmtmd-dspqueue.so libllama-bench-impl-dspqueue.so" + for f in ${fastrpc_libs}; do + [ ! -f ${ab_test_dir}/${f} ] && missing="${missing} ${f}" + done + for f in ${dspqueue_libs}; do + [ ! -f ${ab_test_dir}/${f} ] && missing="${missing} ${f}" + done + if [ -n "${missing}" ]; then + echo "ERROR: AB test backups incomplete, missing:${missing}" + echo "" + echo "Run these two commands first to populate ${ab_test_dir}:" + echo " $0 build # builds mempool/FastRPC ggml-hexagon, backs up *-fastrpc.so" + echo " $0 build_dspqueue # builds dspqueue ggml-hexagon, backs up *-dspqueue.so" + exit 1 + fi + # check DSP skels exist for at least one HTP arch version + local fastrpc_skels=$(ls ${ab_test_dir}/libggml-htp-v*-fastrpc.so 2>/dev/null | wc -l) + local dspqueue_skels=$(ls ${ab_test_dir}/libggml-htp-v*-dspqueue.so 2>/dev/null | wc -l) + if [ ${fastrpc_skels} -eq 0 ] || [ ${dspqueue_skels} -eq 0 ]; then + echo "ERROR: DSP skels missing in ${ab_test_dir}" + echo " fastrpc skels (libggml-htp-v*-fastrpc.so): ${fastrpc_skels} found" + echo " dspqueue skels (libggml-htp-v*-dspqueue.so): ${dspqueue_skels} found" + echo "" + echo "Run these two commands first:" + echo " $0 build # builds mempool/FastRPC DSP skels" + echo " $0 build_dspqueue # builds dspqueue DSP skels" + exit 1 + fi + + echo "==============================================" + echo " AB test: mempool/FastRPC vs dspqueue, ${rounds} rounds each" + echo " model: ${model_path}" + echo " $(date '+%Y-%m-%d %H:%M:%S')" + echo "==============================================" + + # --- mempool/FastRPC phase --- + echo "" + echo "=== [$(date '+%H:%M:%S')] Switching to mempool/FastRPC ===" + adb push ${ab_test_dir}/libggml-hexagon-fastrpc.so ${REMOTE_PATH}/libggml-hexagon.so + adb push ${ab_test_dir}/libggml-fastrpc.so ${REMOTE_PATH}/libggml.so + adb push ${ab_test_dir}/libllama-fastrpc.so ${REMOTE_PATH}/libllama.so + adb push ${ab_test_dir}/libllama-common-fastrpc.so ${REMOTE_PATH}/libllama-common.so + adb push ${ab_test_dir}/libllama-completion-impl-fastrpc.so ${REMOTE_PATH}/libllama-completion-impl.so + adb push ${ab_test_dir}/libllama-server-impl-fastrpc.so ${REMOTE_PATH}/libllama-server-impl.so + adb push ${ab_test_dir}/libmtmd-fastrpc.so ${REMOTE_PATH}/libmtmd.so + adb push ${ab_test_dir}/libllama-bench-impl-fastrpc.so ${REMOTE_PATH}/libllama-bench-impl.so + # skels push as canonical libggml-htp-vXX.so, overwriting any dspqueue skels + for skel in ${ab_test_dir}/libggml-htp-v*-fastrpc.so; do + [ -f "$skel" ] && adb push "$skel" ${REMOTE_PATH}/$(basename "$skel" -fastrpc.so).so + done + # libggml-opencl.so is optional (GGML_OPENCL=OFF by default) + if [ -f ${ab_test_dir}/libggml-opencl-fastrpc.so ]; then + adb push ${ab_test_dir}/libggml-opencl-fastrpc.so ${REMOTE_PATH}/libggml-opencl.so + else + adb shell "rm -f ${REMOTE_PATH}/libggml-opencl.so" + adb shell "rm -f ${REMOTE_PATH}/libggml-vulkan.so" + fi + # legacy name from before the skel unification; clean up leftovers + adb shell "rm -f ${REMOTE_PATH}/libggmldsp-skel-*.so" + + echo "" + echo "========================================" + echo " mempool/FastRPC test (${rounds} runs)" + echo "========================================" + for i in $(seq 1 ${rounds}); do + echo "" + echo "-------- fastrpc run ${i}/${rounds} $(date '+%H:%M:%S') --------" + adb shell "cd ${REMOTE_PATH} && export LD_LIBRARY_PATH=${REMOTE_PATH} && ${REMOTE_PATH}/llama-completion ${running_params} -m ${model_path} -p \"${PROMPT_STRING}\"" + echo "-------- fastrpc run ${i} END --------" + done + + # --- dspqueue phase --- + echo "" + echo "=== [$(date '+%H:%M:%S')] Switching to dspqueue ===" + adb push ${ab_test_dir}/libggml-hexagon-dspqueue.so ${REMOTE_PATH}/libggml-hexagon.so + adb push ${ab_test_dir}/libggml-dspqueue.so ${REMOTE_PATH}/libggml.so + adb push ${ab_test_dir}/libllama-dspqueue.so ${REMOTE_PATH}/libllama.so + adb push ${ab_test_dir}/libllama-common-dspqueue.so ${REMOTE_PATH}/libllama-common.so + adb push ${ab_test_dir}/libllama-completion-impl-dspqueue.so ${REMOTE_PATH}/libllama-completion-impl.so + adb push ${ab_test_dir}/libllama-server-impl-dspqueue.so ${REMOTE_PATH}/libllama-server-impl.so + adb push ${ab_test_dir}/libmtmd-dspqueue.so ${REMOTE_PATH}/libmtmd.so + adb push ${ab_test_dir}/libllama-bench-impl-dspqueue.so ${REMOTE_PATH}/libllama-bench-impl.so + # skels push as canonical libggml-htp-vXX.so, overwriting any fastrpc skels + for skel in ${ab_test_dir}/libggml-htp-v*-dspqueue.so; do + [ -f "$skel" ] && adb push "$skel" ${REMOTE_PATH}/$(basename "$skel" -dspqueue.so).so + done + # libggml-opencl.so is optional (GGML_OPENCL=OFF by default) + if [ -f ${ab_test_dir}/libggml-opencl-dspqueue.so ]; then + adb push ${ab_test_dir}/libggml-opencl-dspqueue.so ${REMOTE_PATH}/libggml-opencl.so + else + adb shell "rm -f ${REMOTE_PATH}/libggml-opencl.so" + adb shell "rm -f ${REMOTE_PATH}/libggml-vulkan.so" + fi + # legacy name from before the skel unification; clean up leftovers + adb shell "rm -f ${REMOTE_PATH}/libggmldsp-skel-*.so" + + echo "" + echo "========================================" + echo " dspqueue test (${rounds} runs)" + echo "========================================" + for i in $(seq 1 ${rounds}); do + echo "" + echo "-------- dspqueue run ${i}/${rounds} $(date '+%H:%M:%S') --------" + adb shell "cd ${REMOTE_PATH} && export LD_LIBRARY_PATH=${REMOTE_PATH} && ${REMOTE_PATH}/llama-completion ${running_params} -m ${model_path} -p \"${PROMPT_STRING}\"" + echo "-------- dspqueue run ${i} END --------" + done + + echo "" + echo "==============================================" + echo " AB test complete $(date '+%Y-%m-%d %H:%M:%S')" + echo "==============================================" + + # Restore current build type libs after AB test. + # AB test leaves dspqueue libs on device (dspqueue is the last phase). + # Without this, subsequent run_llamabench would skip pushing + # libggml-hexagon.so (MD5 matches local fastrpc build), leaving device + # in a mixed state: dspqueue AP lib + fastrpc DSP skels -> error 0x80000406. + local restore_type="" + local last_bt_file="${LOCAL_BUILD_DIR}/.last_deployed_build_type" + if [ -f "${last_bt_file}" ]; then + restore_type=$(cat "${last_bt_file}") + fi + if [ -z "${restore_type}" ]; then + restore_type="hexagon-fastrpc" + fi + echo "" + echo "=== [$(date '+%H:%M:%S')] Restoring build type: ${restore_type} ===" + case "${restore_type}" in + hexagon-fastrpc) + update_fastrpc_libs + ;; + hexagon-dspqueue) + update_dspqueue_libs + ;; + cpu-only) + update_cpu_libs + ;; + esac +} + + +function run_abtest_all() +{ + # Run AB test across all 8 supported models. + # Usage: run_abtest_all [rounds] + # rounds: default 3 (per model); qwen3-9b is hard-capped to 1 (slow + high power, phone gets hot) + # + # Example: + # $0 run_abtest_all + # $0 run_abtest_all 3 + # $0 run_abtest_all 2>&1 | tee log_abtest_all_$(date +%Y%m%d-%H%M%S).txt + + local rounds=3 + if [ $# -ge 1 ]; then + rounds=$1 + fi + + local all_models="gemma4-e2b gemma4-e4b qwen3-2b nanbeige-3b qwen1 minicpm5-1b llama3 qwen3-9b" + local total=8 + local idx=0 + + for model_alias in ${all_models}; do + idx=$((idx + 1)) + # qwen3-9b inference is slow + high power + phone gets hot; cap to 1 round (vs ${rounds} for other models) + local model_rounds=${rounds} + if [ "${model_alias}" = "qwen3-9b" ]; then + model_rounds=1 + echo " NOTE: qwen3-9b -> rounds=1 (slow inference, high power, phone gets hot)" + fi + echo "" + echo "##############################################" + echo " AB test ${idx}/${total}: ${model_alias} (rounds=${model_rounds})" + echo " $(date '+%Y-%m-%d %H:%M:%S')" + echo "##############################################" + run_abtest ${model_rounds} ${model_alias} + done + + echo "" + echo "##############################################" + echo " All AB tests complete $(date '+%Y-%m-%d %H:%M:%S')" + echo "##############################################" +} + + +function run_test-ops() +{ + prog_name=test-backend-ops + prepare_run_on_phone ${prog_name} + + echo "adb shell \"cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && ${REMOTE_PATH}/${prog_name} test\"" + + + adb shell "cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && ${REMOTE_PATH}/${prog_name} test" + +} + + +function run_test-op() +{ + prog_name=test-backend-ops + prog_param="-o ${opname}" + prepare_run_on_phone ${prog_name} + + echo "adb shell cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && ${REMOTE_PATH}/${prog_name} test ${prog_param}" + + echo "\n" + adb shell "cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && ${REMOTE_PATH}/${prog_name} test ${prog_param}" + +} + + +function run_perf-op() +{ + prog_name=test-backend-ops + prepare_run_on_phone ${prog_name} + + echo "adb shell cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && ${REMOTE_PATH}/${prog_name} perf -o ${opname}" + + echo "\n" + adb shell "cd ${REMOTE_PATH} \ + && export LD_LIBRARY_PATH=${REMOTE_PATH} \ + && ${REMOTE_PATH}/${prog_name} perf -o ${opname}" + +} + + +function show_usage() +{ + echo -e "\n" + echo "Usage:" + echo " $0 help" + + echo " $0 build (build the mempool/FastRPC-invoke ggml-hexagon backend for performance comparision)" + echo " $0 build_dspqueue (build the dspqueue ggml-hexagon backend for performance comparison, Qualcomm's official dspqueu-based ggml-hexagon)" + echo " $0 build_armcpu (build Android CPU-only reference for correctness check and troulbeshooting trick issues)" + echo " $0 clean" + + echo " $0 update_fastrpc_libs (push mempool/FastRPC runtime .so from out/ab-test/ to device, for build)" + echo " $0 update_dspqueue_libs (push dspqueue runtime .so from out/ab-test/ to device, for build_dspqueue)" + echo " $0 update_cpu_libs (push CPU-only runtime .so from out/ab-test/ to device, for build_armcpu)" + echo " $0 update_ggml_libs (incremental: push AP-side libs from bin/ to device only; keep DSP skels as-is)" + + echo " $0 run_llamaversion (display llama-cpp version information, e.g. version: 0.2.0-dev (build 11120, commit 9f03708a9), built with Clang 21.0.0 for Android aarch64)" + echo " $0 run_testops" + echo " $0 run_testop ADD/MUL_MAT/FLASH_ATTN_EXT (verify accuracy of ADD/MUL_MAT)" + echo " $0 run_perfop ADD/MUL_MAT/FLASH_ATTN_EXT (verify performance of ADD/MUL_MAT)" + echo -e "\n" + + echo " $0 run_abtest_all [rounds]" + echo " Batch AB test across all 8 models (qwen1 minicpm5-1b llama3 qwen3-2b gemma4-e2b nanbeige-3b gemma4-e4b qwen3-9b)." + echo " rounds: default 3" + echo " Log capture example:" + echo " $0 run_abtest_all 2>&1 | tee log_abtest_all_\$(date +%Y%m%d-%H%M%S).txt" + echo -e "\n" + + echo " $0 run_llamacli [model_alias]" + echo " $0 run_llamabench [model_alias]" + echo " Model aliases for run_llamacli:" + echo " qwen3-2b -> Qwen3.5-2B-Q4_0.gguf" + echo " qwen3-9b -> Qwen3.5-9B-Q4_0.gguf" + echo " gemma4-e2b -> gemma-4-E2B-it-Q4_0.gguf" + echo " gemma4-e4b -> gemma-4-E4B_q4_0-it.gguf" + echo " qwen1 -> qwen1_5-1_8b-chat-q4_0.gguf" + echo " llama3 -> Llama-3.2-1B-Instruct-Q4_0.gguf" + echo " nanbeige-3b -> Nanbeige_Nanbeige4.2-3B-Q4_0.gguf" + echo " minicpm5-1b -> minicpm5-1b-q4_0.gguf" + echo " (default) -> gemma-4-E2B-it-Q4_0.gguf" + echo " Examples:" + echo " $0 run_llamacli/run_llamabench # run gemma4-e2b inference test on an Qualcomm mobile SoC-based Android phone" + echo " $0 run_llamacli/run_llamabench qwen3-2b # test qwen3-2b" + echo " $0 run_llamacli/run_llamabench gemma4-e2b # test gemma4-e2b" + echo " $0 run_llamacli/run_llamabench gemma4-e4b # test gemma4-e4b" +} + + +######## part-4: entry point ######## + +show_pwd + +check_commands_in_host +check_android_phone +check_and_download_ndk +check_and_download_opencl_sdk +check_and_download_hexagon_sdk +check_prebuilt_models + +if [ $# == 0 ]; then + show_usage + exit 1 +elif [ $# == 1 ]; then + if [ "$1" == "-h" ]; then + show_usage + exit 1 + elif [ "$1" == "help" ]; then + show_usage + exit 1 + elif [ "$1" == "update_ggml_libs" ]; then + update_ggml_libs + exit 1 + elif [ "$1" == "update_fastrpc_libs" ]; then + update_fastrpc_libs + exit 0 + elif [ "$1" == "update_dspqueue_libs" ]; then + update_dspqueue_libs + exit 0 + elif [ "$1" == "update_cpu_libs" ]; then + update_cpu_libs + exit 0 + elif [ "$1" == "build" ]; then + build_ggml_hexagon + exit 0 + elif [ "$1" == "build_dspqueue" ]; then + build_ggml_hexagon_dspqueue + exit 0 + elif [ "$1" == "build_armcpu" ]; then + build_armcpu + exit 0 + elif [ "$1" == "clean" ]; then + remove_temp_dir + exit 0 + elif [ "$1" == "run_testops" ]; then + run_test-ops + exit 0 + elif [ "$1" == "run_llamacli" ]; then + run_llamacli + exit 0 + elif [ "$1" == "run_llamaversion" ]; then + run_llamaversion + exit 0 + elif [ "$1" == "run_llamabench" ]; then + run_llamabench + exit 0 + elif [ "$1" == "run_llamacli_all" ]; then + run_llamacli_all + exit 0 + elif [ "$1" == "run_abtest" ]; then + run_abtest + exit 0 + elif [ "$1" == "run_abtest_all" ]; then + run_abtest_all + exit 0 + else + show_usage + exit 1 + fi +elif [ $# == 2 ]; then + if [ "$1" == "run_testop" ]; then + opname=$2 + run_test-op + exit 0 + elif [ "$1" == "run_perfop" ]; then + opname=$2 + run_perf-op + exit 0 + elif [ "$1" == "run_llamacli" ]; then + if [ -z "$(resolve_model_name "$2")" ]; then + echo "ERROR: unknown model alias '$2'. Valid aliases: qwen1 minicpm5-1b llama3 qwen3-2b gemma4-e2b nanbeige-3b gemma4-e4b qwen3-9b" + show_usage + exit 1 + fi + run_llamacli "$2" + exit 0 + elif [ "$1" == "run_llamabench" ]; then + if [ -z "$(resolve_model_name "$2")" ]; then + echo "ERROR: unknown model alias '$2'. Valid aliases: qwen1 minicpm5-1b llama3 qwen3-2b gemma4-e2b nanbeige-3b gemma4-e4b qwen3-9b" + show_usage + exit 1 + fi + run_llamabench "$2" + exit 0 + elif [ "$1" == "run_abtest" ]; then + run_abtest "$2" + exit 0 + elif [ "$1" == "run_abtest_all" ]; then + run_abtest_all "$2" + exit 0 + else + show_usage + exit 1 + fi +elif [ $# == 3 ]; then + if [ "$1" == "run_perfop" ]; then + opname=$2 + run_perf-op + exit 0 + elif [ "$1" == "run_testop" ]; then + opname=$2 + run_test-op + exit 0 + elif [ "$1" == "run_llamacli" ]; then + if [ -z "$(resolve_model_name "$2")" ]; then + echo "ERROR: unknown model alias '$2'. Valid aliases: qwen3-2b, qwen3-9b, gemma4-e2b, gemma4-e4b, qwen1, llama3" + show_usage + exit 1 + fi + run_llamacli "$2" + exit 0 + elif [ "$1" == "run_abtest" ]; then + run_abtest "$2" "$3" + exit 0 + else + show_usage + exit 1 + fi +elif [ $# == 4 ]; then + show_usage + exit 1 +else + show_usage + exit 1 +fi